def evaluate_profile(recommender, users_to_evaluate: set): evaluator = Evaluator() MAP_final, _ = evaluator.evaluateRecommender(recommender, users_to_evaluate) print("MAP-10 score:", MAP_final) return MAP_final
def evaluate_hybrid_weights_validation_kfold(recommender_list, weights, kfold=4, parallel_fit=False, user_group="all", parallelize_evaluation=False): num_cores = multiprocessing.cpu_count() users_to_evaluate_list = [] validation_data_list = [] for i in range(kfold): URM_validation, _, validation_data, _ = Helper().get_kfold_data( kfold)[i] validation_data_list.append(validation_data) if user_group == "cold": users_to_evaluate_list.append( prepare_cold_users(URM_validation, validation_data)) elif user_group == "warm": raise NotImplementedError else: users_to_evaluate_list.append(list(validation_data.keys())) recommender_list[i].fit(**weights) data_list = [] if parallelize_evaluation and kfold <= num_cores: with multiprocessing.Pool(processes=kfold) as p: results = p.starmap( Evaluator().evaluate_recommender_kfold, [(recommender_list[i], users_to_evaluate_list[i], validation_data_list[i]) for i in range(kfold)]) for i in range(len(results)): data_list.append(results[i][0]) p.close() else: for i in range(kfold): recommender = recommender_list[i] # recommender.fit(**weights) MAP, _ = Evaluator().evaluate_recommender_kfold( recommender, users_to_evaluate_list[i], validation_data_list[i]) data_list.append(MAP) MAP_final, variance = compute_mean_and_variance(data_list) print("Variance over k-fold:", variance) print("MAP-10 score:", MAP_final) return MAP_final
def predict(self, spark, tmp, xgb): data = PreProcessor.transVector(tmp, 'features') predictions = xgb.predict( data, -999).map(lambda row: (row['predictions'][1], row['label'])) predictions = predictions.toDF("score", "label") right = predictions.withColumn("idx", monotonically_increasing_id()) left = tmp.select(['name', 'idcard', 'phone']).withColumn("idx", monotonically_increasing_id()) res_df = left.join(right, ['idx'], 'inner').drop('idx') evaluator_handle = Evaluator(spark) auc = evaluator_handle.evaluateAuc(res_df) print("AUC: ", auc) return res_df, auc
def evaluate_hybrid_weights_test(recommender, weights, exclude_users=None): recommender.fit(**weights) MAP_final, _ = Evaluator(test_mode=True).evaluateRecommender( recommender, exclude_users) print("MAP-10 score:", MAP_final) return MAP_final
def evaluate_hybrid_weights_validation(recommender, weights, exclude_users=None): recommender.fit(**weights) MAP_final, _ = Evaluator().evaluateRecommender(recommender, exclude_users) print("MAP-10 score:", MAP_final) return MAP_final
def crossValidationByXGB(self, paramMap): split_num_arr = [] for i in range(self.cv_num): split_num_arr.append(1.0 / self.cv_num) split_data_arr = self.train.randomSplit(split_num_arr, seed=666) split_data_arr.map(lambda x: x.cache()) kfold_res = self.KFold(split_data_arr) train_ks_list = [] val_ks_list = [] train_auc_list = [] val_auc_list = [] for i in range(len(kfold_res)): train_df, validation_df = kfold_res[i] xgb_handle = XGBoostClassifier(config['XGBOOST']) xgbModel = xgb_handle.train(train_df) train_res, train_auc = xgb_handle.predict(self.spark, train_df, xgbModel) validation_res, validation_auc = xgb_handle.predict(self.spark, validation_df, xgbModel) evaluator_handler = Evaluator(self.spark) train_ks = evaluator_handler.evaluateKsCustomized(train_res, "score") val_ks = evaluator_handler.evaluateKsCustomized(validation_res,"score") print("fold"+str(i)+" train ks: {0}".format(train_ks)) print("fold" + str(i) + " val ks: {0}".format(val_ks)) print("fold" + str(i) + " train auc: {0}".format(train_auc)) print("fold" + str(i) + " val auc: {0}".format(validation_auc)) train_ks_list.append(train_ks) train_auc_list.append(train_auc) val_ks_list.append(val_ks) val_auc_list.append(validation_auc) print('----------------avg----------------') print("train average ks is ", np.mean(train_ks_list)) print("val average ks is ", np.mean(val_ks_list)) print("train average auc is ", np.mean(train_auc_list)) print("val average auc is ", np.mean(val_auc_list))
def perform_evaluation_slim(recommender, test_data: dict): """Takes an already fitted recommender and evaluates on test data. If test_mode is false writes the submission""" if not test_data: print("Missing test data! Exiting...") exit(-1) print("Performing evaluation on test set...") MAP_final = 0.0 evaluator = Evaluator() for user in tqdm(test_data.keys()): recommended_items = recommender.recommend(int(user), exclude_seen=True)[:10] relevant_item = test_data[int(user)] MAP_final += evaluator.MAP(recommended_items, relevant_item) MAP_final /= len(test_data.keys()) MAP_final *= 0.665 print("MAP-10 score:", MAP_final) return MAP_final
def perform_evaluation(recommender): """Takes an already fitted recommender and evaluates on test data. If test_mode is false writes the submission""" print("Performing evaluation on test set...") MAP_final = 0.0 evaluator, helper = Evaluator(), Helper() URM_train, eval_data = helper.URM_train_validation, helper.validation_data recommender.fit(URM_train) for user in tqdm(eval_data.keys()): recommended_items = recommender.recommend(int(user), exclude_seen=True) relevant_item = eval_data[int(user)] MAP_final += evaluator.MAP(recommended_items, relevant_item) MAP_final /= len(eval_data.keys()) print("MAP-10 score:", MAP_final) MAP_final *= 0.665 print("MAP-10 public approx score:", MAP_final) return MAP_final
def objective(params): print(params) total_loss = 0 for k in range(4): URM_train, URM_test, validation_data, test_data = Helper().get_kfold_data(4)[k] booster = XGBooster(URM_train, validation_data, HybridElasticNetICFUCF) booster.URM_test = URM_test booster.fit(train_parameters=deepcopy(params)) loss, _ = Evaluator(test_mode=True).evaluate_recommender_kfold(booster, test_data, sequential=True) total_loss += loss total_loss /= 4 print("Map@10 k-fold score:", total_loss) return -total_loss
def train(spark): if config['XGBOOST']['checkpointInitialization'] == 'true': checkpoint_path = config['XGBOOST']['checkpoint_path'] op = os.system("hadoop fs -rmr %s/*" % checkpoint_path) if not op: print("initialize checkpoint successfully.") train_df = Hdfs2Df.readHdfsCsv(spark=spark, data_path=config['TRAIN']['train_path']) test_df = Hdfs2Df.readHdfsCsv(spark=spark, data_path=config['TRAIN']['test_path']) missing = config['XGBOOST']['missing'] train_df = PreProcessor.transColType(train_df, missing) test_df = PreProcessor.transColType(test_df, missing) train, train_col = PreProcessor.transVector(train_df, 'features') test, test_col = PreProcessor.transVector(train_df, 'features') SavaTools.saveModelFeature(train_col, config['TRAIN']['local_model_feature_path']) xgb_handle = XGBoostClassifier(config['XGBOOST']) xgbModel = xgb_handle.trainAndSave(spark, train, config['TRAIN']['hdfs_model_path']) train_res, train_auc = xgb_handle.predict(spark, train, xgbModel) test_res, test_auc = xgb_handle.predict(spark, test, xgbModel) train_res.cache() test_res.cache() evaluator_handle = Evaluator(spark) train_ks = evaluator_handle.evaluateKs(train_res, 'train_res', 'score') train_auc = evaluator_handle.evaluateAuc(train_res, "score") test_ks = evaluator_handle.evaluateKs(test_res, 'test_ks', 'score') test_auc = evaluator_handle.evaluateAuc(test_res, "score") fscore = xgbModel.booster.getFeatureScore() xgb_handle.saveFeatureImportance( train_col, fscore, config['TRAIN']['local_model_feature_weights_path'], train_auc, test_auc, train_ks, test_ks) SavaTools.saveHdfsFile(train_res, config['TRAIN']['train_res_path']) SavaTools.saveHdfsFile(train_res, config['TRAIN']['test_res_path'])
from evaluation.Evaluator import Evaluator if __name__ == "__main__": evaluator = Evaluator() evaluator.split_data_randomly()
def evaluate_on_validation_set(recommender_class, fit_parameters, init_params=None, user_group="all", users_to_evaluate=None, Kfold=0, parallel_fit=False, parallelize_evaluation=False): if init_params is None: init_params = {} evaluator = Evaluator(test_mode=False) if Kfold > 0: MAP_final = 0 fitted_recommenders = [] num_cores = multiprocessing.cpu_count() kfold_data = Helper().get_kfold_data(Kfold) URMs = [data[0] for data in kfold_data] validation_data_list = [data[2] for data in kfold_data] users_to_evaluate_list = [] for data in kfold_data: if user_group == "cold": users_to_evaluate_list.append( prepare_cold_users(data[0], data[2])) elif user_group == "warm": users_to_evaluate_list.append( prepare_warm_users(data[0], data[2])) else: users_to_evaluate_list.append(list(data[2].keys())) if parallel_fit and Kfold <= num_cores: if init_params is not None: raise NotImplementedError( "Can't handle init params in parallel fit") print("Parallelize fitting recommenders...") with multiprocessing.Pool(processes=Kfold) as p: fitted_recommenders = p.map(fit_recommender, [ (recommender_class, URM, recommender_id, fit_parameters, "validation") for URM, recommender_id in zip(URMs, range(len(URMs))) ]) p.close() print("Done!") else: for i in range(Kfold): URM_validation, _, validation_data, _ = Helper( ).get_kfold_data(Kfold)[i] recommender = recommender_class(URM_validation, **init_params) recommender.fit(**fit_parameters) fitted_recommenders.append((recommender, i)) data_list = [] if parallelize_evaluation and Kfold <= num_cores: with multiprocessing.Pool(processes=Kfold) as p: results = p.starmap( Evaluator().evaluate_recommender_kfold, [(recommender, users_to_evaluate_list[recommender_id], validation_data_list[recommender_id]) for recommender, recommender_id in fitted_recommenders ]) for i in range(len(results)): data_list.append(results[i][0]) p.close() else: for recommender, recommender_id in fitted_recommenders: MAP, _ = evaluator.evaluate_recommender_kfold( recommender, users_to_evaluate_list[recommender_id], validation_data_list[recommender_id]) data_list.append(MAP) MAP_final, variance = compute_mean_and_variance(data_list) print("Variance over k-fold:", variance) else: URM_train = Helper().URM_train_validation recommender = recommender_class(URM_train, **init_params) recommender.fit(**fit_parameters) if user_group == "cold": MAP_final, _ = evaluator.evaluate_recommender_on_cold_users( recommender) else: MAP_final, _ = evaluator.evaluateRecommender( recommender, users_to_evaluate) print("MAP-10 score:", MAP_final) return MAP_final
import hyperopt as hp from hyperopt import Trials, fmin, space_eval, STATUS_OK from SLIM_BPR.Cython.SLIM_BPR_Cython import SLIM_BPR_Cython from evaluation.Evaluator import Evaluator from utils.run import RunRecommender import numpy as np from utils.helper import Helper helper = Helper() N_KFOLD = 10 MAX_EVALS = 100 evaluator = Evaluator() ### Step 1 : defining the objective function def objective(params): params["topK"] = int(params["topK"]) params["batch_size"] = int(params["batch_size"]) params["random_seed"] = 1234 params["epochs"] = 30 print("############ Current parameters ############") print(params) loss = -RunRecommender.evaluate_on_validation_set( SLIM_BPR_Cython, params, Kfold=N_KFOLD, parallelize_evaluation=False, user_group="warm")
def LoadMovieLensData(): ml = MovieLens() print("Loading movie ratings...") data = ml.loadMovieLensLatestSmall() print( "\nComputing movie popularity ranks so we can measure novelty later..." ) rankings = ml.getPopularityRanks() return (ml, data, rankings) np.random.seed(0) random.seed(0) # Load up common data set for the recommender algorithms (ml, evaluationData, rankings) = LoadMovieLensData() # Construct an Evaluator to, you know, evaluate them evaluator = Evaluator(evaluationData, rankings) contentKNN = ContentKNNAlgorithm() evaluator.AddAlgorithm(contentKNN, "ContentKNN") # Just make random recommendations Random = NormalPredictor() evaluator.AddAlgorithm(Random, "Random") evaluator.Evaluate(True) evaluator.SampleTopNRecs(ml)
import torch if torch.cuda.is_available(): DEVICE = torch.device('cuda:0') #' else: DEVICE = torch.device('cpu') #'cuda:0' VOCAB_SIZE = 8000 MIN_SEQ_LEN = 5 MAX_SEQ_LEN = 20 BATCH_SIZE = 256 GEN_EMBEDDING_DIM = 256 GEN_HIDDEN_DIM = 256 if __name__ == '__main__': evaluator = Evaluator(vocab_size=VOCAB_SIZE, min_seq_len=MIN_SEQ_LEN, max_seq_len=MAX_SEQ_LEN, batch_size=BATCH_SIZE, device=DEVICE) result = {} for i in range(1, 32): gen = Generator(evaluator.sos_id, evaluator.eou_id, VOCAB_SIZE, GEN_HIDDEN_DIM, GEN_EMBEDDING_DIM, MAX_SEQ_LEN, teacher_forcing_ratio=0) model_path = 'generator_checkpoint' + str(i) + '.pth.tar' data = torch.load(model_path, map_location='cpu') gen.load_state_dict(data['state_dict']) gen.decoder = TopKDecoder(gen.decoder, 5) gen.to(DEVICE) print('Evaluating ' + model_path) result[i] = evaluator.evaluate_embeddings(gen) print(result[i])
) rankings = azr.getPopularityRanks() #rankings = defaultdict(int) return (azr, data, rankings) # Load up common data set for the recommender algorithms (ml, evaluationData, rankings) = LoadMovieLensData() #(azr,evaluationData, rankings) = LoadAmazonData() # Construct an Evaluator to, you know, evaluate them ######### our evaluation code ######### evaluator = Evaluator(evaluationData, rankings, doTopN=True) SVDAlgorithm = SVDpp() evaluator.AddAlgorithm(SVDAlgorithm, "SVD++") #SVDppAlgorithm = SVDpp() #evaluator.AddAlgorithm(SVDppAlgorithm, "SVD++") evaluator.Evaluate() #evaluator.SampleTopNRecs(ml) ### buit-in fonction for evaluation #######" #algo = SVD() # Run 3-fold cross-validation and print results #cross_validate(algo, evaluationData, measures=['RMSE', 'MAE'], cv=3, verbose=True)
discriminator = discriminator.Discriminator(DIS_EMBEDDING_DIM,\ DIS_HIDDEN_DIM, VOCAB_SIZE, MAX_SEQ_LEN, device=DEVICE).to(DEVICE) else: discriminator = discriminator_LM2.LM(DIS_EMBEDDING_DIM, VOCAB_SIZE, device=DEVICE).to(DEVICE) if DISCRIMINATOR_CHECKPOINT: discriminator.load_state_dict( torch.load(DISCRIMINATOR_CHECKPOINT, map_location=DEVICE)) dis_optimizer = optim.Adagrad(discriminator.parameters(), lr=DISCRIMINATOR_LR) evaluator = Evaluator(vocab_size=VOCAB_SIZE, min_seq_len=MIN_SEQ_LEN, max_seq_len=MAX_SEQ_LEN, batch_size=BATCH_SIZE_TESTING, device=DEVICE) # Define critic and dual optimizer if AC: critic = critic.Critic(DIS_EMBEDDING_DIM, DIS_HIDDEN_DIM, VOCAB_SIZE, MAX_SEQ_LEN, device=DEVICE).to(DEVICE) AC_optimizer = optim.Adagrad([{ 'params': actor.parameters(), 'lr': ACTOR_LR }, { 'params': critic.parameters(),
def main(): """ Track objects. """ visualize_steps = True evaluate = True downsampling_factor = 2 mot_sequence = '02' #02, 04 or 09 # ---------------- DATA READING ------------------- print('Data reading...') # Read image sequence into frames list video_path = f'test_data/MOT17-{mot_sequence}-raw.webm' frame_list = FrameList(video_path) frame_list.read_frames() # ------------- BACKGROUND MODELING --------------- print('Background modeling...') # For each frame in frame_list.frames, create likelihood images & suppress shadows. filter_type = 'gaussian' background_model = BackgroundModel(filter_type) background_model.create_binary_images(frame_list.frames, visualize=visualize_steps) background_model.suppress_shadows(frame_list.frames, visualize=visualize_steps) plt.close('all') # ------------- FOREGROUND SEGMENTATION ----------- print('Foreground segmentation...') # For each frames in frame_list.frames, remove noise & create objects foreground_segmentation = ForegroundSegmentation(frame_list) foreground_segmentation.remove_noise() foreground_segmentation.label_image(min_box_area=1500, visualize=visualize_steps) # ------------------- TRACKER --------------------- print('Tracking...') # Create ObjectMatcher and Predictor objects used by Tracker object_matcher = ObjectMatcher(match_algorithm='simple') predictor = Predictor(Q0=1, R0=1, P0=np.array([1, 1, 0.01, 0.01]), T=1) # Create Tracker object tracker = Tracker(object_matcher, predictor) plt.figure() # Match objects between frames 2 by 2 print('Number of frames =', len(frame_list.frames)) for i in range(1, len(frame_list.frames) - 1): tracker.match_objects(frame_list.frames[i], frame_list.frames[i + 1]) if (i % 20 == 0): print(round(i / len(frame_list.frames) * 100), '%') plt.close('all') TrackerVisualizer(frame_list) # ------------------ EVALUATION ------------------- print('Evaluating...') # This will only work with MOT17 dataset 02, 04 or 09 if (video_path.startswith('test_data/MOT17') and evaluate): evaluator = Evaluator(frame_list, f'test_data/MOT17-{mot_sequence}-gt.txt') evaluator.read_gt(downsampling_factor=downsampling_factor) #evaluator.generate_eval_csv(mot_sequence, downsampling_factor=downsampling_factor) tp_sum, fp_sum, fn_sum, id_switches, precision, recall, avg_tp_overlap = evaluator.generate_metrics( ) print(f'tp_sum: {tp_sum}') print(f'fn_sum: {fn_sum}') print(f'fp_sum: {fp_sum}') print(f'id_switches: {id_switches}') print(f'precision: {precision}') print(f'recall: {recall}') print(f'avg_tp_overlap: {avg_tp_overlap}') # Visualize ground truth plt.close('all')
# compute the scores using the dot product user_profile = self.URM[user_id] scores = user_profile.dot(self.similarity_matrix).toarray().ravel() if exclude_seen: scores = self.filter_seen(user_id, scores) # rank items ranking = scores.argsort()[::-1] return ranking[:at] if __name__ == '__main__': evaluator = Evaluator() helper = Helper() MAP_final = 0.0 # TODO URM_data contains the whole dataset and not the train set URM_all = helper.convert_URM_to_csr(helper.URM_data) URM_train, URM_test, target_users_test, test_data = split_train_test( URM_all, 0.8) recommender = SLIMRecommender(URM_train) recommender.fit(epochs=1000) # Load target users # target_users_test = helper.load_target_users_test() # relevant_items = helper.load_relevant_items()
"num_factors": hp.hp.uniformint('num_factors', 1, 5000), "batch_size": hp.hp.choice('batch_size', [32, 64, 128, 256, 512]), "learning_rate": hp.hp.loguniform('learning_rate', 1e-6, 1e-2), "epochs": hp.hp.uniformint('epochs', 1, 1000), "validation_every_n": hp.hp.choice("validation_every_n", [10]), "validation_metric": hp.hp.choice('validation_metric', ['MAP']), "lower_validations_allowed": hp.hp.choice('lower_validations_allowed', [2]), "evaluator_object": hp.hp.choice('evaluator_object', [Evaluator(test_mode=False)]) } if __name__ == '__main__': ### step 3 : storing the results of every iteration bayes_trials = Trials() MAX_EVALS = 20 # Optimize best = fmin( fn=objective, space=MF_space, algo=hp.tpe.suggest, max_evals=MAX_EVALS, trials=bayes_trials, verbose=True,