class Server(): def __init__(self, conf, data): self.conf = conf self.data = data self.fed_clients = self.conf.fed_clients self.syft_clients = self.conf.syft_clients self.init() def init(self): print("-> initial server") self.helper = Helper(self.conf) self.eval = Evaluation(self.conf, self.data) self.model = Model(num_features=self.conf.num_features, num_classes=self.conf.num_classes).to(self.conf.device) def run(self): # training process for t in range(self.conf.num_round): start = time.time() # random select client curr_client_ids = np.random.choice(self.conf.num_clients, self.conf.num_per_round, replace=False) print(f"\n\n-> t={t} selected_client={curr_client_ids}") # client update spdz_weights = {} for uid in curr_client_ids: spdz_weights[uid] = self.fed_clients[uid].update(self.model.state_dict()) if not self.conf.fed_horizontal["encrypt_weight"]: print("-> client send params to server without shared_encrypt") new_weights = self.aggregation(spdz_weights) self.model.copy_params(new_weights) stop = time.time() print("-> end round:{:d} using:{:.2f}s".format(t, float(stop-start))) # evluation self.eval.eval_test(self.model) def aggregation(self, spdz_weights): if len(spdz_weights) == 1: return spdz_weights[0] if self.conf.fed_aggregate == "avg": return self.helper.weights_avg(self.model.state_dict(), spdz_weights, fix_precision=self.conf.fed_horizontal["encrypt_weight"])
def __init__(self, URM_train, mode="validation"): self.URM_train = URM_train # Init single recommenders self.top_pop = HybridUserCBFRegionalTopPop(URM_train) self.SSLIM = MultiThreadSSLIM_ElasticNet(URM_train) self.UCF = UserCollaborativeFilter(URM_train) self.user_cbf = UserCBF(URM_train) # Get the cold users list self.cold_users, _ = Helper().compute_cold_warm_user_ids(URM_train) # Fit the single recommenders, this saves time in case of changing just the relative weights self.top_pop.fit() self.SSLIM.fit(**SSLIM_parameters) self.UCF.fit(**UCF_parameters) self.user_cbf.fit(**user_cbf_parameters)
def fit(self, topK=410, shrink=0, normalize=True, similarity="cosine", bm_25_norm=False): self.topK = topK self.shrink = shrink self.normalize = normalize self.similarity = similarity if bm_25_norm: self.URM_train = Helper().bm25_normalization(self.URM_train) self.W_sparse = self.compute_similarity_matrix(self.URM_train, self.shrink, self.topK, self.normalize, self.similarity)
def evaluate_fgan_loss(inet, fnet, dataloader, nactors): print('Evaluate Fusion Network') inet.eval() fnet.eval() top1 = AverageMeter() top5 = AverageMeter() for batch_idx, (inputs, targets) in enumerate(dataloader): z_missed, z_missed_g, target_missed, _ = batch_process( inet, fnet, inputs, targets, nactors) out_missed_g = inet.module.classifier(z_missed_g) prec1, prec5 = Helper.accuracy(out_missed_g, target_missed, topk=(1, 5)) top1.update(prec1.item(), target_missed.size(0)) top5.update(prec5.item(), target_missed.size(0)) del z_missed, z_missed_g, target_missed, _ return top1.avg, top5.avg
def search_comments(self, q, page=0, page_size=DEFAULT_PAGE_SIZE, startdate=None, enddate=None): session = self.db_helper.Session() try: news = session.query(News).filter( News.news_id == self.news_id).first() query = session.query(Comments).filter( Comments.news_id == self.news_id).filter( Comments.comment.contains(q)) t_query = session.query(func.count('*').label('total')).filter( Comments.news_id == self.news_id).filter( Comments.comment.contains(q)) if startdate and startdate != '': query = query.filter( Comments.comment_time >= Helper.get_date(startdate)) t_query = t_query.filter( Comments.comment_time >= Helper.get_date(startdate)) if enddate and enddate != '': query = query.filter( Comments.comment_time <= Helper.get_date(enddate)) t_query = t_query.filter( Comments.comment_time <= Helper.get_date(enddate)) query = query.order_by(Comments.comment_time.desc()) results = self.db_helper.query(query, page=page, page_size=page_size) total_comments = int(t_query[0][0]) pages = int(total_comments / page_size) if total_comments % page_size == 0 else int( total_comments / page_size) + 1 return { 'dates': { 'start': (Helper.get_date(startdate).strftime('%Y-%m-%d') if startdate and startdate != '' else ''), 'end': (Helper.get_date(enddate).strftime('%Y-%m-%d') if enddate and enddate != '' else '') }, 'pages': pages, 'comments': [result.to_dict() for result in results], 'news': news.to_dict() } except Exception as ex: self.logger.error("Exception occurred when searching comments. ", ex) return {'pages': 0, 'comments': []} finally: session.close()
def __parse_comments(self, response): for sel in response.xpath("//div[@id='comments']/ul/li"): try: item = response.meta['item'] item['comment_id'] = int( sel.xpath("@data-cid").extract_first().strip()) item['comment'] = sel.xpath( "div[@class='comment']/p[1]/span[1]//text()" ).extract_first().strip() item['comment_time'] = Helper.parse_comment_time( sel.xpath( "div[@class='comment']/h3[1]/span[2]/span[2]/text()"). extract_first().strip()) yield item except Exception as ex: self.logger.error( f"Exception occurred when parsing comment with response {response}", ex) yield None
def objective(params): print(params) total_loss = 0 for k in range(4): URM_train, URM_test, validation_data, test_data = Helper().get_kfold_data(4)[k] booster = XGBooster(URM_train, validation_data, HybridElasticNetICFUCF) booster.URM_test = URM_test booster.fit(train_parameters=deepcopy(params)) loss, _ = Evaluator(test_mode=True).evaluate_recommender_kfold(booster, test_data, sequential=True) total_loss += loss total_loss /= 4 print("Map@10 k-fold score:", total_loss) return -total_loss
def run(recommender_class, fit_parameters, init_params=None, at=10, submission_path="data/recommendation_matrix_to_submit.csv"): # Helper contains methods to convert URM in CSR if init_params is None: init_params = {} URM_all = Helper().URM_csr recommender = recommender_class(URM_all, **init_params) recommender.fit(**fit_parameters) RunRecommender.write_submission(recommender, submission_path=submission_path, at=at)
def init(self, *weight): print("-> initial server") self.helper = Helper(self.conf) # creat homomorphic encryption key pair # TODO # init weight if not len(weight): weight = [None, None] # setup model for uid, party in self.party.items(): self.model[uid] = Model(party.num_features, party.num_output, weight=weight[uid]).to(self.conf.device) self.optimizer[uid] = optim.SGD(self.model[uid].parameters(), lr=self.conf.learning_rate, momentum=self.conf.momentum)
def __init__(self, URM_train, mode="dataset"): self.URM_train = URM_train # Init single recommenders self.item_cbf = ItemCBF(URM_train) self.item_cf = ItemCollaborativeFilter(URM_train) self.top_pop = HybridUserCBFRegionalTopPop(URM_train) self.SLIM = MultiThreadSLIM_ElasticNet(URM_train) self.rp3 = RP3betaRecommender(URM_train) # Get the cold users list self.cold_users, _ = Helper().compute_cold_warm_user_ids(URM_train) # Fit the single recommenders, this saves time in case of changing just the relative weights self.item_cbf.fit(**item_cbf_parameters) self.item_cf.fit(**item_cf_parameters) self.rp3.fit(**rp3_parameters) self.top_pop.fit() self.SLIM.fit(**SLIM_parameters)
def __init__( self, URM_train, mode="validation", ): self.URM_train = URM_train # Init single recommenders self.user_cf = UserCollaborativeFilter(URM_train) self.item_cf = ItemCollaborativeFilter(URM_train) self.top_pop = TopPopRecommender(URM_train) self.als = AlternatingLeastSquare(URM_train) # Get the cold users list self.cold_users, _ = Helper().compute_cold_warm_user_ids(URM_train) # Fit the single recommenders, this saves time in case of changing just the relative weights self.user_cf.fit(**user_cf_parameters) self.item_cf.fit(**item_cf_parameters) self.top_pop.fit() self.als.fit(**als_parameters)
def get_all_userfeedbacks(self): """ Get all users'feedbacks from table user's feedback """ # Read User Feedback from csv user_feedback = UserFeedback(**self.config) ### Read all feedbacks from table user's feedback criteria = {} df_userfb = user_feedback.get_user_feedback(criteria) df_userfb = df_userfb.dropna() # Filter by Feedback_App name; Feedback_App= YOUTUBE_GENERAL try: ## Remove from column RTT: the values Blank space, NaN, Zero, #NULL!, +Inf df_userfb[self.config['COL']['RTT']] = df_userfb[ self.config['COL']['RTT']].apply(lambda x: self.__to_number(x)) ## Recording the inconsistent instances index dropIx = df_userfb[df_userfb[self.config['COL']['RTT']] == -1].index ## Dropping these instances from the dataset: df_userfb.drop(dropIx, inplace=True) df_userfb.reset_index(inplace=True) # Rename columns to match the columns name in model df_userfb = Helper.rename_df_columnname(df_userfb, self.config['COL']) # Save for testing purpose filename = self.config['OUT_DIR'] + 'all_ufb_except_unknown.csv' df_userfb.to_csv(filename) except IOError as error: print(error) return df_userfb
def fit(self, top_pop_weight=0.02139131367609725, topK=765, shrink=6, normalize=True, bm_25_norm=False, similarity="jaccard", suppress_interactions=False, asymmetric_alpha=0.5): if bm_25_norm: self.URM_train = Helper().bm25_normalization(self.URM_train) self.toppop = RegionalTopPopRecommender(self.URM_train) self.user_cbf = UserCBF(self.URM_train) self.user_cbf.fit(topK=topK, shrink=shrink, normalize=normalize, similarity=similarity, suppress_interactions=suppress_interactions, asymmetric_alpha=asymmetric_alpha) self.toppop.fit() self.toppop_weight = top_pop_weight
def import_todo_list(request): if request.method == 'POST': result = Helper.message() try: file = request.FILES.get('csv_file', None) try: Helper.validate_file_extension(file, '.csv') except ValidationError: result[ 'message'] = 'File type is not valid.Please select a csv file.' decoded_file = file.read().decode('ISO-8859-1').replace( '', '').splitlines() file_content = list( csv.reader(decoded_file, delimiter=';', quotechar='"')) header = file_content.pop(0) todo_index = Helper.column_index('Todo', header) status_index = Helper.column_index('Status', header) authenticated_user_id = Helper.get_session(request).user_id if todo_index is not -1 and status_index is not -1: for r in file_content: try: Todo.objects.create( user_id=authenticated_user_id, text=r[todo_index].strip(), is_completed=False if 'Not Completed' in r[status_index] else True) except Exception as ex: print(ex) result = Helper.message_success(text='Records are created.') else: result['message'] = 'File format is not valid.' except Exception as ex: print(ex) result['message'] = str(ex) return JsonResponse(result) return render(request, 'pages/todo/import_form.html', {})
def perform_evaluation(recommender): """Takes an already fitted recommender and evaluates on test data. If test_mode is false writes the submission""" print("Performing evaluation on test set...") MAP_final = 0.0 evaluator, helper = Evaluator(), Helper() URM_train, eval_data = helper.URM_train_validation, helper.validation_data recommender.fit(URM_train) for user in tqdm(eval_data.keys()): recommended_items = recommender.recommend(int(user), exclude_seen=True) relevant_item = eval_data[int(user)] MAP_final += evaluator.MAP(recommended_items, relevant_item) MAP_final /= len(eval_data.keys()) print("MAP-10 score:", MAP_final) MAP_final *= 0.665 print("MAP-10 public approx score:", MAP_final) return MAP_final
def __init__( self, URM_train, mode="validation", ): self.URM_train = URM_train # Init single recommenders self.user_cf = UserCollaborativeFilter(URM_train) self.item_cf = ItemCollaborativeFilter(URM_train) self.top_pop = TopPopRecommender(URM_train) self.SLIM = MultiThreadSLIM_ElasticNet(URM_train) # self.user_based_cbf = UserBasedCBF(URM_train) # self.item_based_cbf = ItemBasedCBF(URM_train) # Get the cold users list self.cold_users, _ = Helper().compute_cold_warm_user_ids(URM_train) # Fit the single recommenders, this saves time in case of changing just the relative weights self.user_cf.fit(**user_cf_parameters) self.item_cf.fit(**item_cf_parameters) self.top_pop.fit() self.SLIM.fit(**SLIM_parameters)
def plot_reconstruction(model_name, sample_path, inet, fnet, dataloader, nactors, concat_input=False): print('Evaluate fusion network: ' + model_name) inet.eval() fnet.eval() corrects = [0, 0, 0] criterionL1 = torch.nn.L1Loss(reduction='mean') iters = iter(dataloader) (inputs, targets) = iters.next() inputs = Variable(inputs).cuda() targets = Variable(targets).cuda() N, C, H, W = inputs.shape M = N // nactors N = M * nactors inputs = inputs[:N, ...].view(M, nactors, C, H, W) targets = targets[:N, ...].view(M, nactors) # fusion inputs x_g = inputs.view(M, nactors * C, H, W) x_fused_g = rescale(fnet(x_g)) # Z _, z, _ = inet(inputs.view(M * nactors, C, H, W)) z = z.view((M, nactors, z.shape[1], z.shape[2], z.shape[3])) target_missed = targets[:, 0] z_missed = z[:, 0, ...] z_rest = z[:, 1:, ...].sum(dim=1) z_fused = z.mean(dim=1) _, z_fused_g, _ = inet(x_fused_g) z_missed_g = nactors * z_fused_g - z_rest # classification out_missed_g = inet.module.classifier(z_missed_g) out_missed = inet.module.classifier(z_missed) # evaluation _, y = torch.max(out_missed.data, 1) _, y_g = torch.max(out_missed_g.data, 1) corrects[0] = y.eq(target_missed.data).sum().cpu().item() corrects[1] = y_g.eq(target_missed.data).sum().cpu().item() corrects[2] = y_g.eq(y.data).sum().cpu().item() # inverse x_fused_g = inet.module.inverse(z_fused_g) x_fused = inet.module.inverse(z_fused) x_missed = inet.module.inverse(z_missed) x_missed_g = inet.module.inverse(z_missed_g) # visualize Helper.save_images(x_fused_g, sample_path, model_name, 'x_fused_g', 0) Helper.save_images(x_fused, sample_path, model_name, 'x_fused', 0) Helper.save_images(x_missed_g, sample_path, model_name, 'x_missed_g', 0) Helper.save_images(x_missed, sample_path, model_name, 'x_missed', 0) print("L1: {:.4f}".format( criterionL1(torch.tanh(x_fused_g), torch.tanh(x_fused)))) print(np.asarray(corrects) / M) from IPython import embed embed()
# HybridUCFICFRecommender = HybridUCFICFRecommender(Helper().URM_train_test) recommender_class = HybridSSLIMICFUCFRP3Beta N_KFOLD = 10 kfold = False parallel_fit = True if kfold > 0: MAP_final = 0 recommender_list = [] print("Getting Kfold splits...\n") kfold_data = Helper().get_kfold_data(N_KFOLD) print("Done!\n") for i in range(N_KFOLD): print("Fitting recommender", i+1, "...\n") recommender_list.append(HybridSSLIMICFUCFRP3Beta(kfold_data[i][0], multithreaded=True)) print("\nCompleted!\n") else: hybrid = HybridSSLIMICFUCFRP3Beta(Helper().URM_train_validation) # Step 1 : defining the objective function def objective(params): print("\n############## New iteration ##############\n", params) if kfold: loss = - RunRecommender.evaluate_hybrid_weights_validation_kfold(recommender_list, params, kfold=N_KFOLD, parallelize_evaluation=kfold, parallel_fit=False)
recommender_class = Hybrid recommenders = [MultiThreadSLIM_ElasticNet, RP3betaRecommender, ItemCBF, AlternatingLeastSquare] N_KFOLD = 6 kfold = True if kfold > 0: MAP_final = 0 recommender_list = [] print("Getting Kfold splits...\n") kfold_data = Helper().get_kfold_data(N_KFOLD) print("Done!\n") for i in range(N_KFOLD): print("Fitting recommender", i+1, "...\n") recommender_list.append(recommender_class(kfold_data[i][0], recommenders)) print("\nCompleted!\n") else: hybrid = recommender_class(Helper().URM_train_validation, recommenders) # Step 1 : defining the objective function def objective(params): print("\n############## New iteration ##############\n", params) params = {"weights": params} if kfold:
class Evaluator: def __init__(self, split_size=0.8): self.helper = Helper() self.URM_data = self.helper.URM_data # Matrix containing full original training data self.URM_test = pd.DataFrame(columns=["playlist_id", "track_id" ]) # Matrix containing test data self.URM_train = None # Matrix containing reduced training data, at the beginning it is equal to the test data (then it will be removed) self.target_playlists_test = None # New list of playlists for which MAP will be computed self.split_size = split_size # Percentage of training set that will be kept for training purposes (1-split_size is the size of the test set) def split_data_randomly(self): # Instantiate an array containing the playlists to be put in new target_playlist_file playlists_list = np.asarray(list(self.URM_data.playlist_id)) URM_csr = self.helper.convert_URM_data_to_csr() # Will contain target playlist ids of the test file self.target_playlists_test = np.empty(shape=playlists_list.shape) # Group by the URM just to have faster search grouped_by_playlist_URM = self.URM_data.groupby( 'playlist_id', as_index=True).apply(lambda x: list(x['track_id'])) # Number of tuples of the test set, used for the upper bound over the counter test_dimensions = len(playlists_list) * (1 - self.split_size) for count in tqdm(range(int(test_dimensions))): tracks_less_than_10 = True # Don't consider items with less than 10 elements (would not be good for testing since Kaggle evaluates with MAP10) while tracks_less_than_10: # Choose a random index between 0 and the length of URM's number of playlists random_index_to_be_put_in_test = randint( 0, len(playlists_list) - 1) candidate_playlist = playlists_list[ random_index_to_be_put_in_test] # If there the playlist has fewer than 10 songs, we have to discard it if len(URM_csr[candidate_playlist].indices ) >= NUMBER_OF_TRACKS_TEST_FILE: # Append random playlist to playlist test set self.target_playlists_test[count] = playlists_list[ random_index_to_be_put_in_test] # np array containing all the tracks contained in a selected playlist track_list_10 = np.asarray( list(grouped_by_playlist_URM[candidate_playlist])) # Keep just 10 tracks for each playlist playlist_tracks_list = track_list_10[: NUMBER_OF_TRACKS_TEST_FILE] # Create a tuple to be appended in URM_test matrix and append it URM_test_tuple = pd.DataFrame({ "playlist_id": [int(self.target_playlists_test[count])], "track_id": [str(playlist_tracks_list)] }) self.URM_test = self.URM_test.append(URM_test_tuple, ignore_index=True) # Exit while loop tracks_less_than_10 = False # Format data of test_data.csv correctly and save it self.URM_test["track_id"] = self.URM_test["track_id"].str.strip(' []') self.URM_test["track_id"] = self.URM_test["track_id"].replace( '\s+', ' ', regex=True) self.URM_test.to_csv(os.path.join(ROOT_PROJECT_PATH, "data/test_data.csv"), index=False) print("URM_test created, now creating URM_train") self.target_playlists_test = np.asarray(list( self.URM_test.playlist_id)) self.URM_train = self.URM_data for playlist in tqdm(self.target_playlists_test): #print(self.URM_data.loc[self.URM_data["playlist_id"] != playlist]) self.URM_train = self.URM_train.loc[~( self.URM_train["playlist_id"] == playlist)] self.URM_train.to_csv(os.path.join(ROOT_PROJECT_PATH, "data/train_data.csv"), index=False) target_playlists_new_csv_file = open( os.path.join(ROOT_PROJECT_PATH, "data/target_playlists_test.csv"), "w") target_playlists_new_csv_file.write("playlist_id\n") for playlist in tqdm(self.target_playlists_test): target_playlists_new_csv_file.write(str(int(playlist)) + "\n")
def __init__(self, URM): self.URM_train = URM self.helper = Helper()
class AssetCBF: def __init__(self, URM): self.URM_train = URM self.helper = Helper() def compute_similarity_cbf(self, ICM, top_k, shrink, normalize=True, similarity="cosine"): # Compute similarities for weighted features recommender similarity_object = Compute_Similarity_Python(ICM.T, shrink=shrink, topK=top_k, normalize=normalize, similarity=similarity) w_sparse = similarity_object.compute_similarity() return w_sparse def fit(self, topK, shrink): self.topK = topK self.shrink = shrink # Load ICMs from helper self.ICM_asset = self.helper.bm25_normalization( self.helper.load_icm_asset()) # Computing SMs self.SM_asset = self.compute_similarity_cbf(self.ICM_asset, top_k=self.topK, shrink=self.shrink) def compute_scores(self, user_id): users_list_train = self.URM_train[user_id] scores_asset = users_list_train.dot(self.SM_asset).toarray().ravel() return scores_asset def recommend(self, user_id, at=10, exclude_seen=True): # Compute scores of the recommendation scores = self.compute_scores(user_id) # Filter to exclude already seen items if exclude_seen: scores = self.filter_seen(user_id, scores) recommended_items = np.argsort(scores) recommended_items = np.flip(recommended_items, axis=0) return recommended_items[:at] def filter_seen(self, user_id, scores): start_pos = self.URM_train.indptr[user_id] end_pos = self.URM_train.indptr[user_id + 1] user_profile = self.URM_train.indices[start_pos:end_pos] scores[user_profile] = -np.inf return scores
import hyperopt as hp from hyperopt import Trials, fmin, STATUS_OK from Hybrid_ALS_SLIMElastic import HybridSSLIMUCF from utils.run import RunRecommender from utils.helper import Helper hybrid = HybridSSLIMUCF(Helper().URM_train_validation, mode="validation") ### Step 1 : defining the objective function def objective(params): print("Current parameters:") print(params) loss = - RunRecommender.evaluate_hybrid_weights_validation(hybrid, params) return loss als_space = { "SLIM_weight": hp.hp.uniform('SLIM_weight', 0.7, 1.0), } if __name__ == '__main__': ### step 3 : storing the results of every iteration bayes_trials = Trials() MAX_EVALS = 50 # Optimize best = fmin(fn=objective, space=als_space, algo=hp.tpe.suggest, max_evals=MAX_EVALS, trials=bayes_trials, verbose=True)
def defaults(request): return { 'is_admin': Helper.is_admin(request), 'user': Helper.get_session(request), }
def init(self): print("-> initial server") self.helper = Helper(self.conf) self.eval = Evaluation(self.conf, self.data) self.model = Model(num_features=self.conf.num_features, num_classes=self.conf.num_classes).to(self.conf.device)
class CBFRecomender: def __init__(self, knn_artist=25, knn_album=45, shrink_artist=0, shrink_album=8, weight_artist=0.15): self.knn_artist = knn_artist self.knn_album = knn_album self.shrink_artist = shrink_artist self.shrink_album = shrink_album self.weight_artist = weight_artist self.helper = Helper() def compute_similarity_cbf(self, ICM, top_k, shrink, normalize=True, similarity="cosine"): #Compute similarities for weighted features recommender similarity_object = Compute_Similarity_Python(ICM.T, shrink=shrink, topK=top_k, normalize=normalize, similarity=similarity) w_sparse = similarity_object.compute_similarity() return w_sparse def fit(self, URM): # URM Loading self.URM = URM # Load ICMs from helper self.ICM_artist = self.helper.load_icm_artist() self.ICM_artist = self.helper.tfidf_normalization(self.ICM_artist) self.ICM_artist = self.helper.sklearn_normalization(self.ICM_artist) self.ICM_album = self.helper.load_icm_album() self.ICM_album = self.helper.bm25_normalization(self.ICM_album) # Computing SMs self.SM_artist = self.compute_similarity_cbf(self.ICM_artist, top_k=self.knn_artist, shrink=self.shrink_artist) self.SM_album = self.compute_similarity_cbf(self.ICM_album, top_k=self.knn_album, shrink=self.shrink_album) def compute_scores(self, playlist_id): tracks_list_train = self.URM[playlist_id] scores_artist = tracks_list_train.dot(self.SM_artist).toarray().ravel() scores_album = tracks_list_train.dot(self.SM_album).toarray().ravel() weight_album = 1 - self.weight_artist scores = (scores_artist * self.weight_artist) + (scores_album * weight_album) return scores def recommend(self, playlist_id, at=10, exclude_seen=True): # Compute scores of the recommendation scores = self.compute_scores(playlist_id) # Filter to exclude already seen items if exclude_seen: scores = self.filter_seen(playlist_id, scores) recommended_items = np.argsort(scores) recommended_items = np.flip(recommended_items, axis=0) return recommended_items[:at] def filter_seen(self, playlist_id, scores): start_pos = self.URM.indptr[playlist_id] end_pos = self.URM.indptr[playlist_id + 1] user_profile = self.URM.indices[start_pos:end_pos] scores[user_profile] = -np.inf return scores
import hyperopt as hp from hyperopt import Trials, fmin, space_eval, STATUS_OK from SLIM_BPR.Cython.SLIM_BPR_Cython import SLIM_BPR_Cython from evaluation.Evaluator import Evaluator from utils.run import RunRecommender import numpy as np from utils.helper import Helper helper = Helper() N_KFOLD = 10 MAX_EVALS = 100 evaluator = Evaluator() ### Step 1 : defining the objective function def objective(params): params["topK"] = int(params["topK"]) params["batch_size"] = int(params["batch_size"]) params["random_seed"] = 1234 params["epochs"] = 30 print("############ Current parameters ############") print(params) loss = -RunRecommender.evaluate_on_validation_set( SLIM_BPR_Cython, params, Kfold=N_KFOLD, parallelize_evaluation=False, user_group="warm")
def eval_inv(model, testloader, sample_path, model_name, num_epochs, nactors, debug=False): model_name = "{}_epoch_{}".format(model_name, num_epochs) print('Evaluate Invertibility on ', model_name) model.eval() # loss criterion = torch.nn.MSELoss(reduction='sum') corrects = [0, 0, 0] # correct-x, correct-x-hat, match total = 0 in_shapes = -1 for batch_idx, (inputs, targets) in enumerate(testloader): # print(batch_idx) targets = Variable(targets).cuda() inputs = Variable(inputs).cuda() batch_size, C, H, W = inputs.shape total += batch_size out, z_input, _ = model(inputs) x = inputs.clone().unsqueeze(0).expand( nactors - 1, batch_size, C, H, W).contiguous().view( (nactors - 1) * batch_size, C, H, W) x = x[torch.randperm(x.shape[0]), :] _, z_c, _ = model(x) z_c = z_c.view(nactors - 1, batch_size, z_input.shape[1], z_input.shape[2], z_input.shape[3]).sum(dim=0) z_fused = (z_input + z_c) / nactors x_fused = model.module.inverse(z_fused) _, z_fused_hat, _ = model(x_fused) z_hat = z_fused_hat * nactors - z_c # invert x_inv = model.module.inverse(z_input) x_inv_hat = model.module.inverse(z_hat) # classification out_hat = model.module.classifier(z_hat) _, y = torch.max(out.data, 1) _, y_hat = torch.max(out_hat.data, 1) corrects[0] += y.eq(targets.data).sum().cpu().item() corrects[1] += y_hat.eq(targets.data).sum().cpu().item() corrects[2] += y_hat.eq(y.data).sum().cpu().item() # save samples if batch_idx == 0: Helper.save_images(x_inv, sample_path, model_name, 'inv', batch_idx) Helper.save_images(x_inv_hat, sample_path, model_name, 'inv_hat', batch_idx) Helper.save_images(x_fused, sample_path, model_name, 'fused', batch_idx) del out, out_hat, z_c, z_fused, z_hat, z_fused_hat, inputs, x_inv, x_inv_hat, targets, y, y_hat # print('\t {} images of {} pixels'.format(total, in_shapes)) corrects = 100 * np.asarray(corrects) / total print('\t Correctly classified: X {:.4f} X_hat {:.4f} Match {:.4f}'. format(corrects[0], corrects[1], corrects[2])) return corrects[0]
from hyperopt import hp, tpe, fmin, Trials from hyperopt import STATUS_OK, STATUS_FAIL import pickle import os import traceback __author__ = "Guillaume Chevalier" __copyright__ = "Copyright 2017, Guillaume Chevalier" __license__ = "MIT License" __notice__ = ( "Some further edits by Guillaume Chevalier are made on " "behalf of Vooban Inc. and belongs to Vooban Inc. ") # See: https://github.com/Vooban/Hyperopt-Keras-CNN-CIFAR-100/blob/master/LICENSE" h = Helper('topomaps_RT_100/train/combined/', 'topomaps_RT_100/test/combined/', 'results/RT_100') space = { # This loguniform scale will multiply the learning rate, so as to make # it vary exponentially, in a multiplicative fashion rather than in # a linear fashion, to handle his exponentialy varying nature: 'lr_rate_mult': hp.loguniform('lr_rate_mult', -0.5, 0.5), # L2 weight decay: 'l2_weight_reg_mult': hp.loguniform('l2_weight_reg_mult', -1.3, 1.3), # Batch size fed for each gradient update 'batch_size': hp.quniform('batch_size', 40, 128, 10), # Number of EPOCHS 'epochs': hp.quniform('epochs', 10, 120, 10), # Choice of optimizer: 'optimizer': hp.choice('optimizer', ['Adam', 'Nadam', 'RMSprop']), # Uniform distribution in finding appropriate dropout values, conv layers
from utils.provider import FusionDataset from configs import args ############ Settings ############## TRAIN_FRACTION = 0.8 cudnn.benchmark = True torch.cuda.manual_seed(args.seed) device = torch.device("cuda:0") args.fnet_name = "fnet_{}_{}_{}_{}".format(args.fname, args.dataset, args.iname, args.nactors) args.inet_name = "inet_{}_{}".format(args.dataset, args.iname) checkpoint_dir = os.path.join(args.save_dir, 'checkpoints') args.inet_save_dir = os.path.join(checkpoint_dir, args.inet_name) args.fnet_save_dir = os.path.join(checkpoint_dir, args.fnet_name) Helper.try_make_dir(args.save_dir) Helper.try_make_dir(checkpoint_dir) Helper.try_make_dir(args.fnet_save_dir) Helper.try_make_dir(args.inet_save_dir) if args.dataset == 'cifar10': args.input_nc = 3 args.output_nc = 3 in_shape = (3, 32, 32) else: args.input_nc = 1 args.output_nc = 1 in_shape = (1, 32, 32) fusion_data_dir = os.path.join(args.data_dir, "fusion/{}_{}_{}".format(args.dataset, args.iname, args.nactors)) train_path = os.path.join(fusion_data_dir, 'train.npy')