def speedTestAndEval(): rankNet=RankNet(64) rankNetMod=RankNetMod(64) file='data/Fold2/train.txt' queries=load_queries(file,64) start=timer() rankNet.train_with_queries(queries,4) print("- Took %.2f sec to train rankNet " % (timer() - start)) start=timer() rankNetMod.train_with_queries(queries,4) print("- Took %.2f sec to train rankNetMod " % (timer() - start)) print('----Now lets evaluate--------') ndcg=NDCG(1) testQueries=load_queries('data/Fold2/test.txt',64) for name,ranker in zip(['rankNet','rankNetMod'],[rankNet,rankNetMod]): r=[] for q in testQueries: rel=getRankedList(ranker,q) r.append(ndcg.run(rel,max_c=np.sum(rel))) print('mNDCG for '+name+": ") print(np.mean(r))
def experiment(experiment_type): print '- Running', experiment_type n_features = 64 # Implements 5-Folds validation kfold_ndcg = [] for i in xrange(1, 6): ranker = LambdaRankHW(n_features, type=experiment_type) n_epochs = 5 def query_ndcg(q): scores = ranker.score(q).flatten() labels = q.get_labels() return ndcg(zip(*sorted(zip(labels, scores), key=itemgetter(1), reverse=True))[0]) for j in xrange(1, 6): if i == j: continue queries = query.load_queries('HP2003/Fold%d/train.txt' % j, n_features) ranker.train_with_queries(queries, n_epochs) queries = query.load_queries('HP2003/Fold%d/train.txt' % i, n_features) kfold_ndcg.append(np.mean([query_ndcg(q) for q in queries])) print "- mNDCG: %.3f" % kfold_ndcg[-1] print "- Average mNDCG: %.3f" % np.average(kfold_ndcg)
def run_exp(): ### CHANGE ALGORITHM HERE ### ## Possible values: POINTWISE, PAIRWISE, LISTWISE ALGORITHM = PAIRWISE # Change here for other algos FEATURES = 64 EPOCHS = 5 num_folds = 5 NDCG_AVG = [] folds = [] for fold in range(1, num_folds + 1): # Ranker for this cross-validation. ranker = LambdaRankHW(FEATURES, algorithm=ALGORITHM) # Fold NDCG scores fold_scores = [] for cross_fold in range(1, num_folds + 1): if fold == cross_fold: continue # Current fold taining queries training_queries = query.load_queries( "HP2003/Fold%d/train.txt" % cross_fold, FEATURES) ranker.train_with_queries(training_queries, EPOCHS) fold_scores.append( np.mean( [compute_query_NDCG(q, ranker) for q in training_queries])) # Load test queries test_queries = query.load_queries("HP2003/Fold%d/test.txt" % fold, FEATURES) # Compute and add NDCG on test set NDCG_AVG.append( np.mean([compute_query_NDCG(q, ranker) for q in test_queries])) # Also store NDCG scores on fold to plot them folds.append(fold_scores) # Save and compute average over all folds list_file_name = ALGORITHM + "_NDCGS.npy" average_file_name = ALGORITHM + "_average_NDCG.npy" all_fold_scores = ALGORITHM + "_allscores.npy" np.save(list_file_name, np.array(NDCG_AVG)) np.save(all_fold_scores, np.array(folds)) total_average = np.average(NDCG_AVG) np.save(average_file_name, total_average)
def queryRanker(self): #Extract the high frequency queries from the training_queries HighFreqQueries = [] training_queries = queryClass.load_queries(self.path_train, self.feature_count) test_queries = queryClass.load_queries(self.path_test, self.feature_count) #loop through all queries in the training set for index in training_queries.get_qids(): highQuery = training_queries.get_query(index) #only keep the frequent queries if(len(highQuery.__labels__) > self.minFreqCount): HighFreqQueries.append(highQuery) print "found "+ str(len(HighFreqQueries)) + " high frequency queries" #build the query-ranker dictionary BestRanker = queryRankers() user_model = environment.CascadeUserModel(self.clickModel) evaluation2 = evaluation.NdcgEval() #test_queries = query.load_queries(sys.argv[2], feature_count) print "Read in training and testing queries" #for every query learn the best ranker and save it to the dictionary iter=0 for highQuery in HighFreqQueries: ran=random.random() iter=iter+1 if ran<self.threshold: print str(iter*100/len(HighFreqQueries))+"%" for i in xrange(self.rankersPerQuery): learner = retrieval_system.ListwiseLearningSystem(self.feature_count, '-w random -c comparison.ProbabilisticInterleave -r ranker.ProbabilisticRankingFunction -s 3 -d 0.1 -a 0.01') BestRanker.addInitRank(highQuery.get_qid(),learner.get_solution().w) q = highQuery for t in range(self.iterationCount): l = learner.get_ranked_list(q) c = user_model.get_clicks(l, q.get_labels()) s = learner.update_solution(c) e = evaluation2.evaluate_all(s, test_queries) BestRanker.add(highQuery.get_qid(),learner.get_solution().w) BestRanker.addList(highQuery.get_qid(),l) BestRanker.addEval(highQuery.get_qid(),e) #save the dictionary to a file ('bestRanker.p') paths=self.path_train.split('/') name=paths[1] #pickle.dump(BestRanker, open( "QueryData/"+name+".data", "wb" ) ) pickle.dump(BestRanker, open( "QueryData/"+self.dataset+str(self.iterationCount)+".data", "wb" ) ) test = pickle.load( open( "QueryData/"+self.dataset+str(self.iterationCount)+".data", "rb" ) ) print test.query_ranker.values()
def perform(): # init test_num_features = 64 queries = query.load_queries('../../data/NP2004/Fold1/test.txt', 64) bi = comparison.BalancedInterleave() user_model = environment.CascadeUserModel('--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') # make rankers rankers = [] for i in range(0,5): rankers.append(ranker.ProbabilisticRankingFunction('3', 'random', 64, init=parseRanker('../../data/features64/ranker-0'+str(i)+'.txt'),sample='sample_unit_sphere')) # main loop for N in [100,1000,10000]: pref_matrix = [[0 for x in xrange(5)] for x in xrange(5)] for iter in range(0,N): q = queries[random.choice(queries.keys())] for i in range(0,5): for j in range (0,5): if i!=j: list, context = bi.interleave(rankers[i], rankers[j], q, 10) clicks = user_model.get_clicks(list,q.get_labels()) result = bi.infer_outcome(list,context,clicks,q) if result < 0: pref_matrix[i][j] += 1 else: pref_matrix[i][j] = 0.50 pref_matrix = generateProbabilityMatrix(pref_matrix,N) printMatrix(pref_matrix) print 'Best ranker is ' + '0' + str(getBestRanker(pref_matrix)) + ' (N = ' + str(N) + ').' print 'done!'
def Train(self): print "Loading Data" clusterData=pickle.load(open( self.clusterDataPath, "rb" ) ) feature_count=len(clusterData.clusterToRanker[0][0]) training_queries = queryClass.load_queries(self.testQueries, feature_count) ranker=pickle.load( open( self.rankerPath ) ) """ testWeights=str(clusterData.clusterToRanker[0][0]) testWeights=testWeights.replace("[", "") testWeights=testWeights.replace("]", "") weights = np.array([float(num) for num in testWeights.split(",")]) ranker_tie="random" ranker_args="3" sample_send="sample_unit_sphere" ranker=rankerClass.ProbabilisticRankingFunction(ranker_args, ranker_tie, feature_count, sample=sample_send, init=testWeights) """ X=[] Y=[] max=100 #max number of docs in the ranking #print clusterData.queryToCluster.keys() #print training_queries.keys() print "Loading training objects" for qid in clusterData.queryToCluster: query = training_queries.get_query(qid) ranker.init_ranking(query) docIds=ranker.get_ranking() iter=0 for docId in docIds: if iter>max: break features=query.get_feature_vector(docId) X.append(features) Y.append(clusterData.queryToCluster[qid][0]) iter=iter+1 #X = [[0, 0], [1, 1]] #y = [0, 1] X=np.array(X) Y=np.array(Y) print "Training" clf = svm.SVC() clf.fit(X, Y) if not os.path.exists("Classifier"): os.makedirs("Classifier") paths=self.clusterDataPath.split('/') name=paths[len(paths)-1] parts=name.split('.') name=parts[0] pickle.dump(clf, open( "Classifier/"+name+".data", "wb" ) )
def main(scale, schema, runs, warm, end): queries = load_queries() logger.info("Loaded %d queries", len(queries)) db = "SNOWFLAKE_SAMPLE_DATA" if schema else "TCPH_SCHEMALESS" schema_name = f"TPCH_SF{scale}" schema_id = "SCHEMA" if schema else "SCHEMALESS" with conn_vw(name = f'TEMPORARY_{schema_id}_SF{scale}', size='MEDIUM') as conn: logger.info("Running on database %s", db) cur = conn.cursor() cur.execute(f"USE DATABASE {db};") logger.info("Running on schema %s", db) cur.execute(f"USE SCHEMA {schema_name};") logger.info("Disabling result set cache!") cur.execute("ALTER SESSION SET USE_CACHED_RESULT = FALSE;") cur.close() filename = f"results_{schema_id}_SF{scale}.csv" filepath = os.path.join("results", filename) logger.info("Writing results to %s", filepath) with open(filepath, "w") as f: header = ["Query"]+[f"Run {i+1}" for i in range(runs)] + ["Average", "Standard Deviation"] write_row(f, header) for q in queries[:end]: timings, avg, std = time_query(q, conn, runs=runs, warmups=warm) timings = list(timings) timings += [avg / 1000.0, std / 1000.0] timings = [f"{q.num}"] + [f"{x:.06f}" for x in timings] write_row(f, timings)
def Save(self): print "Loading Data" training_queries = queryClass.load_queries(self.testQueries, self.feature_count) ranker = pickle.load(open(self.rankerPath)) max = 100 #max number of docs in the ranking #print clusterData.queryToCluster.keys() #print training_queries.keys() BestRanker = queryFeatures() print "Loading training objects" i = 0 for query in training_queries: #print str(i*100/len(training_queries))+"%" i = i + 1 #query = training_queries.get_query(qid) ranker.init_ranking(query) docIds = ranker.get_ranking() iter = 0 for docId in docIds: if iter > max: break iter = iter + 1 features = query.get_feature_vector(docId) BestRanker.add(query.get_qid(), features) #print features #BestRanker.addFeaturesToQid([float(i) for i in features],query.get_qid()) pickle.dump(BestRanker, open("QueryData/" + self.dataset + ".data", "wb"))
def groupRanker(self): #Extract the high frequency queries from the training_queries clusterData=pickle.load(open( self.clusterDataPath, "rb" ) ) queryData= self.queryData HighFreqQueries = [] training_queries = queryClass.load_queries(self.path_train, self.feature_count) test_queries = queryClass.load_queries(self.path_test, self.feature_count) #loop through all queries in the training set #build the query-ranker dictionary BestRanker = queryRankers() user_model = environment.CascadeUserModel(self.clickModel) evaluation2 = evaluation.NdcgEval() #test_queries = query.load_queries(sys.argv[2], feature_count) print "Read in training and testing queries" #for every query learn the best ranker and save it to the dictionary iter=0 learner=[0]*len(clusterData.clusterToRanker.keys()) for cluster in clusterData.clusterToRanker: learner[cluster] = retrieval_system.ListwiseLearningSystem(self.feature_count, '-w random -c comparison.ProbabilisticInterleave -r ranker.ProbabilisticRankingFunction -s 3 -d 0.1 -a 0.01') for t in range(self.iterationCount): q = training_queries[random.choice(training_queries.keys())] temp=(float(np.sum(clusterData.queryToCluster[q.get_qid()])))/(float(len(clusterData.queryToCluster[q.get_qid()]))) temp=int(temp+0.5) cluster=temp #cluster=clusterData.queryToCluster[q.get_qid()][0] iter=iter+1 if iter%200==0: print str(iter*100/self.iterationCount)+"%" l = learner[cluster].get_ranked_list(q) c = user_model.get_clicks(l, q.get_labels()) s = learner[cluster].update_solution(c) #e = evaluation2.evaluate_all(s, test_queries) for cluster in clusterData.clusterToRanker: clusterData.clusterToRanker[cluster]=[learner[cluster].get_solution().w.tolist()] #save the dictionary to a file ('bestRanker.p') paths=self.path_train.split('/') name=paths[1] #pickle.dump(BestRanker, open( "QueryData/"+name+".data", "wb" ) ) pickle.dump(clusterData, open( "ClusterData/"+self.dataset+".data", "wb" ) )
def compareSystems(vali_queries,classifierPath,basic_ranker_path,clust_data_path,data,click): print "-Loading Data-" clf = pickle.load( open( classifierPath ) ) basic_ranker=pickle.load( open( basic_ranker_path ) ) clusterData=pickle.load(open(clust_data_path)) queryData=pickle.load(open(data)) ranker_tie="random" feature_count=basic_ranker.feature_count ranker_args="3" arg_str="" sample_send="sample_unit_sphere" iterations=100 rankers=[0]*2 rankers[0]=basic_ranker user_model = environment.CascadeUserModel(click) training_queries = query.load_queries(vali_queries, feature_count) compar_interleave=ProbabilisticInterleave(None) first_win=0 print "-Calculating-" for i in range(iterations): if i%(iterations/10)==0: print str(float(i)*100/float(iterations))+"%" q = training_queries.get_query(random.choice(queryData.query_ranker.keys())) test=queryData.query_ranker[q.get_qid()][0] testWeights=str(test) testWeights=testWeights.replace("[", "") testWeights=testWeights.replace("]", "") weights = np.array([float(num) for num in testWeights.split(",")]) print len(weights) ranker_tie="random" ranker_args="3" sample_send="sample_unit_sphere" rankers[1]=rankerClass.ProbabilisticRankingFunction(ranker_args, ranker_tie, feature_count, sample=sample_send, init=testWeights) l, a = compar_interleave.interleave(rankers[0], rankers[1], q, 10) c = user_model.get_clicks(l, q.get_labels()) o = compar_interleave.infer_outcome(l, a, c, q) if(o<0): first_win+=1 elif(o==0): coin=random.random() if(coin>0.5): first_win+=1 result_com=float(first_win)/float(iterations) print "Basic ranker win rate:"+ str(result_com)
def OneFoldTest(self,folder,model,epochs): testFile=folder+"/test.txt" testQueries=load_queries(testFile,64) trainFile=folder+"/train.txt" trainQueries=load_queries(trainFile,64) model.train_with_queries(trainQueries,epochs) ndcgs=[] for q in testQueries: ndcgs.append(self.ndcg.run(getRankedList(model,q))) return (np.mean(ndcgs))
def compareSystems(vali_queries, classifierPath, basic_ranker_path, clust_data_path, click): print "-Loading Data-" clf = pickle.load(open(classifierPath)) basic_ranker = pickle.load(open(basic_ranker_path)) clusterData = pickle.load(open(clust_data_path)) ranker_tie = "random" feature_count = basic_ranker.feature_count ranker_args = "3" arg_str = "" sample_send = "sample_unit_sphere" iterations = 100 rankers = [0] * 2 rankers[0] = basic_ranker user_model = environment.CascadeUserModel(click) training_queries = query.load_queries(vali_queries, feature_count) compar_interleave = ProbabilisticInterleave(None) second_win = 0 second_win_or_e = 0 generic_win = 0 equal = 0 print "-Calculating-" for i in range(iterations): if i % (iterations / 10) == 0: print str(float(i) * 100 / float(iterations)) + "%" q = training_queries[random.choice(training_queries.keys())] rankers[1] = classifier.getRanker(clf, basic_ranker, q, clusterData) l, a = compar_interleave.interleave(rankers[0], rankers[1], q, 10) c = user_model.get_clicks(l, q.get_labels()) o = compar_interleave.infer_outcome(l, a, c, q) if (o > 0): second_win += 1 second_win_or_e += 1 elif (o == 0): equal += 1 coin = random.random() if (coin > 0.5): second_win_or_e += 1 else: generic_win += 1 result_com = float(second_win_or_e) / float(iterations) result_win = float(second_win) / float(iterations) result_win_generic = float(generic_win) / float(iterations) print "Our ranker win rate (with random choice if result was equal):" + str( result_com) print "Our ranker win rate:" + str(result_win) print "Generic ranker win rate:" + str(result_win_generic) print "Number win ours:" + str(second_win) print "Number win generic:" + str(generic_win) print "Number equal:" + str(equal) print "Total number iterations:" + str(iterations)
def compareSystems(vali_queries,classifierPath,basic_ranker_path,clust_data_path,click): print "-Loading Data-" clf = pickle.load( open( classifierPath ) ) basic_ranker=pickle.load( open( basic_ranker_path ) ) clusterData=pickle.load(open(clust_data_path)) ranker_tie="random" feature_count=basic_ranker.feature_count ranker_args="3" arg_str="" sample_send="sample_unit_sphere" iterations=100 rankers=[0]*2 rankers[0]=basic_ranker user_model = environment.CascadeUserModel(click) training_queries = query.load_queries(vali_queries, feature_count) compar_interleave=ProbabilisticInterleave(None) second_win=0 second_win_or_e=0 generic_win=0 equal = 0 print "-Calculating-" for i in range(iterations): if i%(iterations/10)==0: print str(float(i)*100/float(iterations))+"%" q = training_queries[random.choice(training_queries.keys())] rankers[1]=classifier.getRanker(clf, basic_ranker,q,clusterData) l, a = compar_interleave.interleave(rankers[0], rankers[1], q, 10) c = user_model.get_clicks(l, q.get_labels()) o = compar_interleave.infer_outcome(l, a, c, q) if(o>0): second_win+=1 second_win_or_e+=1 elif(o==0): equal += 1 coin=random.random() if(coin>0.5): second_win_or_e+=1 else: generic_win+=1 result_com=float(second_win_or_e)/float(iterations) result_win=float(second_win)/float(iterations) result_win_generic=float(generic_win)/float(iterations) print "Our ranker win rate (with random choice if result was equal):"+ str(result_com) print "Our ranker win rate:"+ str(result_win) print "Generic ranker win rate:"+ str(result_win_generic) print "Number win ours:" + str(second_win) print "Number win generic:" + str(generic_win) print "Number equal:" + str(equal) print "Total number iterations:" + str(iterations)
def run(self,mainFolder,epochs): folders = get_immediate_subdirectories(mainFolder) #folders = random.sample(get_immediate_subdirectories(mainFolder),2) ndcgs=[[] for i in range(len(self.models))] elapsed=np.zeros(len(self.models)) # for timing for i,folder in enumerate(folders): print("fold "+str(i+1)) trainFile=mainFolder+folder+"/train.txt" train_queries=load_queries(trainFile,64) testFile=mainFolder+folder+"/test.txt" testQueries=load_queries(testFile,64) for i,model in enumerate(self.models): now=timer() model.train_with_queries(train_queries,epochs) elapsed[i]+=timer()-now # evaluation ndcgs[i]+=self.__evalaute(testQueries,model) return ([np.mean(n) for n in ndcgs],elapsed)
def __init__(self, Path, feature_count, queriesPath): self.Path = Path self.queriesPath = queriesPath self.feature_count = feature_count self.listOrEucl = int(raw_input("Euclidean(0) or list(1) distance (answer with 0 or 1)?")) if(self.listOrEucl): print 'using list distance' else: print 'using euclidean distance' print "Reading in queries" self.training_queries = queryClass.load_queries(queriesPath, feature_count) print "Calculating distance" self.calculate()
def experiment(n_epochs, measure_type, num_features, num_folds): best_ranker = None best_val_score = 0 store_res = [] for fold in range(1, num_folds + 1): # Load queries from the corresponding fold print('Loading train queries') train_queries = query.load_queries( os.path.normpath('./HP2003/Fold%d/train.txt' % fold), num_features) print('Loading val queries') val_queries = query.load_queries( os.path.normpath('./HP2003/Fold%d/vali.txt' % fold), num_features) # Creates the S matrix (as described in paper) using scipy.sparse print('Creating the S Matrix') S = create_S_matrix({**train_queries, **val_queries}) # Creates a new ranker ranker = LambdaRankHW(num_features, measure_type) # Stores the statistics for each epoch res = ranker.train_with_queries(train_queries, n_epochs, val_queries, S) final_val_score = res[-1][ 'val_mndcg'] # validation mNDCG after all epochs # keep a running maximum of val mNDCG (also the best ranker) if final_val_score > best_val_score: best_ranker = ranker best_val_score = final_val_score # Stores the results for the current fold store_res.append(res) return store_res, ranker
def __init__(self, Path, feature_count, queriesPath): self.Path = Path self.queriesPath = queriesPath self.feature_count = feature_count self.listOrEucl = int( raw_input( "Euclidean(0) or list(1) distance (answer with 0 or 1)?")) if (self.listOrEucl): print 'using list distance' else: print 'using euclidean distance' print "Reading in queries" self.training_queries = queryClass.load_queries( queriesPath, feature_count) print "Calculating distance" self.calculate()
def perform(): # init test_num_features = 64 queries = query.load_queries('../../data/NP2004/Fold1/test.txt', 64) bi = comparison.BalancedInterleave() user_model = environment.CascadeUserModel( '--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') # make rankers rankers = [] for i in range(0, 5): rankers.append( ranker.ProbabilisticRankingFunction( '3', 'random', 64, init=parseRanker('../../data/features64/ranker-0' + str(i) + '.txt'), sample='sample_unit_sphere')) # main loop for N in [100, 1000, 10000]: pref_matrix = [[0 for x in xrange(5)] for x in xrange(5)] for iter in range(0, N): q = queries[random.choice(queries.keys())] for i in range(0, 5): for j in range(0, 5): if i != j: list, context = bi.interleave(rankers[i], rankers[j], q, 10) clicks = user_model.get_clicks(list, q.get_labels()) result = bi.infer_outcome(list, context, clicks, q) if result < 0: pref_matrix[i][j] += 1 else: pref_matrix[i][j] = 0.50 pref_matrix = generateProbabilityMatrix(pref_matrix, N) printMatrix(pref_matrix) print 'Best ranker is ' + '0' + str( getBestRanker(pref_matrix)) + ' (N = ' + str(N) + ').' print 'done!'
reps = 5 nrqueries = 500 #alphas = [0.75, 1.0, 1.25, 1.5] #deltas = [1.75, 2.0, 2.25] #reps = 5 #nrqueries = 500 factor_k1 = 13.3 factor_k3 = .1 user_model = environment.CascadeUserModel( '--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') evaluator = evaluation.NdcgEval() training_queries = query.load_queries(sys.argv[1], 64) test_queries = query.load_queries(sys.argv[2], 64) def run(alpha, delta): results = [] for _ in range(reps): #learner = retrieval_system.ListwiseLearningSystem(64, '-w random -c comparison.ProbabilisticInterleave -r ranker.ProbabilisticRankingFunction -s 3 ranker.model.BM25 -d %.2f -a %.2f' % (delta, alpha)) learner = retrieval_system.ListwiseLearningSystemWithCandidateSelection( 64, '--num_repetitions 10 --num_candidates 6 --history_length 10 --select_candidate select_candidate_repeated -w random -c comparison.ProbabilisticInterleave --ranker ranker.ProbabilisticRankingFunction --ranker_args 3 ranker.model.BM25 -d %s -a %s --anneal 50' % (delta, alpha)) for _ in range(nrqueries): q = training_queries[random.choice(training_queries.keys())] l = learner.get_ranked_list(q) c = user_model.get_clicks(l, q.get_labels())
metrics = [] scores = {} for metric in "evaluation.NdcgEval", "evaluation.LetorNdcgEval": eval_class = get_class(metric) eval_metric = eval_class() metrics.append(eval_metric) scores[eval_metric.__class__.__name__] = {} for cutoff in cutoffs: scores[eval_metric.__class__.__name__][cutoff] = [] # load all queries test_queries = {} for fold in range(1, 6): test_file = "".join((args.test_dir, str(fold))) test_file = os.path.join(test_file, args.test_file) qs = load_queries(test_file, args.feature_count) test_queries[fold] = qs # process all experiments for all metrics count_experiments = 0 for experiment in args.experiment_dirs: print "%% %s" % experiment count_runs = 0 count_experiments += 1 # process all folds and run files for fold_id in sorted(os.listdir(experiment)): fold = os.path.join(experiment, fold_id) fold_id = int(fold_id) if not os.path.isdir(fold): continue for filename in sorted(os.listdir(fold)):
import sys, random try: import include, pickle except: pass import retrieval_system, environment, evaluation, query learner = retrieval_system.ListwiseLearningSystem(64, '-w random -c comparison.ProbabilisticInterleave -r ranker.ProbabilisticRankingFunction -s 3 -d 0.1 -a 0.01') user_model = environment.CascadeUserModel('--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') evaluation = evaluation.NdcgEval() training_queries = query.load_queries(sys.argv[1], 64) test_queries = query.load_queries(sys.argv[2], 64) i=0 for i in range(10): q = training_queries[random.choice(training_queries.keys())] l = learner.get_ranked_list(q) c = user_model.get_clicks(l, q.get_labels()) s = learner.update_solution(c) print i i=i+1 print evaluation.evaluate_all(s, test_queries) pickle.dump(learner.ranker, open( "QueryData/"+"generalRanker"+".data", "wb" ) )
def __init__(self, args_str=None): # parse arguments parser = argparse.ArgumentParser(description=""" Construct and run a learning experiment. Provide either the name of a config file from which the experiment configuration is read, or provide all arguments listed under Command line. If both are provided the config file is ignored.""", prog=self.__class__.__name__) # option 1: use a config file file_group = parser.add_argument_group("FILE") file_group.add_argument("-f", "--file", help="Filename of the config " "file from which the experiment details" " should be read.") # option 2: specify all experiment details as arguments detail_group = parser.add_argument_group("DETAILS") detail_group.add_argument( "-i", "--training_queries", help="File from which to load the training queries (svmlight " "format).") detail_group.add_argument( "-j", "--test_queries", help="File from which to load the test queries (svmlight format).") detail_group.add_argument( "-c", "--feature_count", type=int, help="The number of features included in the data.") detail_group.add_argument( "-r", "--num_runs", type=int, help="Number of runs (how many times to repeat the experiment).") detail_group.add_argument("-q", "--num_queries", type=int, help="Number of queries in each run.") detail_group.add_argument("-u", "--user_model", help="Class implementing a user model.") detail_group.add_argument( "-v", "--user_model_args", help="Arguments for initializing the user model.") # the retrieval system maintains ranking functions, accepts queries and # generates result lists, and in return receives user clicks to learn # from detail_group.add_argument( "-s", "--system", help="Which system to use (e.g., pairwise, listwise).") detail_group.add_argument("-a", "--system_args", help="Arguments for " "the system (comparison method, learning " "algorithm and parameters...).") detail_group.add_argument( "-o", "--output_dir", help="(Empty) directory for storing output generated by this" " experiment. Subdirectory for different folds will be generated" "automatically.") detail_group.add_argument("--output_dir_overwrite", default="False") detail_group.add_argument( "-p", "--output_prefix", help="Prefix to be added to output filenames, e.g., the name of " "the data set, fold, etc. Output files will be stored as " "OUTPUT_DIR/PREFIX-RUN_ID.txt.gz") detail_group.add_argument("-e", "--experimenter", help="Experimenter type.") # run the parser if args_str: args = parser.parse_known_args(args_str.split())[0] else: args = parser.parse_known_args()[0] # determine whether to use config file or detailed args self.experiment_args = None if args.file: config_file = open(args.file) self.experiment_args = yaml.load(config_file) config_file.close() # overwrite with command-line options if given for arg, value in vars(args).items(): if value: self.experiment_args[arg] = value else: self.experiment_args = vars(args) # workaround - check if we have all the arguments needed if not ("training_queries" in self.experiment_args and "test_queries" in self.experiment_args and "feature_count" in self.experiment_args and "num_runs" in self.experiment_args and "num_queries" in self.experiment_args and "user_model" in self.experiment_args and "user_model_args" in self.experiment_args and "system" in self.experiment_args and "system_args" in self.experiment_args and "output_dir" in self.experiment_args): parser.print_help() sys.exit("Missing required arguments, please check the program" " arguments or configuration file. %s" % self.experiment_args) # set default values for optional arguments if not "query_sampling_method" in self.experiment_args: self.experiment_args["query_sampling_method"] = "random" if not "output_dir_overwrite" in self.experiment_args: self.experiment_args["output_dir_overwrite"] = False if not "experimenter" in self.experiment_args: self.experiment_args[ "experimenter"] = "experiment.LearningExperiment" if not "evaluation" in self.experiment_args: self.experiment_args["evaluation"] = "evaluation.NdcgEval" if not "processes" in self.experiment_args: self.experiment_args["processes"] = 0 # locate or create directory for the current fold if not os.path.exists(self.experiment_args["output_dir"]): os.makedirs(self.experiment_args["output_dir"]) elif not(self.experiment_args["output_dir_overwrite"]) and \ os.listdir(self.experiment_args["output_dir"]): # make sure the output directory is empty raise Exception( "Output dir %s is not an empty directory. " "Please use a different directory, or move contents out " "of the way." % self.experiment_args["output_dir"]) logging.basicConfig(format='%(asctime)s %(module)s: %(message)s', level=logging.INFO) logging.info("Arguments: %s" % self.experiment_args) for k, v in sorted(self.experiment_args.iteritems()): logging.info("\t%s: %s" % (k, v)) config_bk = os.path.join(self.experiment_args["output_dir"], "config_bk.yml") logging.info("Backing up configuration to: %s" % config_bk) config_bk_file = open(config_bk, "w") yaml.dump(self.experiment_args, config_bk_file, default_flow_style=False) config_bk_file.close() # load training and test queries training_file = self.experiment_args["training_queries"] test_file = self.experiment_args["test_queries"] self.feature_count = self.experiment_args["feature_count"] logging.info("Loading training data: %s " % training_file) self.training_queries = load_queries(training_file, self.feature_count) logging.info("... found %d queries." % self.training_queries.get_size()) logging.info("Loading test data: %s " % test_file) self.test_queries = load_queries(test_file, self.feature_count) logging.info("... found %d queries." % self.test_queries.get_size()) # initialize and run the experiment num_run times self.num_runs = self.experiment_args["num_runs"] self.output_dir = self.experiment_args["output_dir"] self.output_prefix = self.experiment_args["output_prefix"] self.experimenter = get_class(self.experiment_args["experimenter"])
def __init__(self, args_str=None): # parse arguments parser = argparse.ArgumentParser(description=""" Construct and run a learning experiment. Provide either the name of a config file from which the experiment configuration is read, or provide all arguments listed under Command line. If both are provided the config file is ignored.""", prog=self.__class__.__name__) # option 1: use a config file file_group = parser.add_argument_group("FILE") file_group.add_argument("-f", "--file", help="Filename of the config " "file from which the experiment details" " should be read.") # option 2: specify all experiment details as arguments detail_group = parser.add_argument_group("DETAILS") detail_group.add_argument("-i", "--training_queries", help="File from which to load the training queries (svmlight " "format).") detail_group.add_argument("-j", "--test_queries", help="File from which to load the test queries (svmlight format).") detail_group.add_argument("-c", "--feature_count", type=int, help="The number of features included in the data.") detail_group.add_argument("-r", "--num_runs", type=int, help="Number of runs (how many times to repeat the experiment).") detail_group.add_argument("-q", "--num_queries", type=int, help="Number of queries in each run.") detail_group.add_argument("-u", "--user_model", help="Class implementing a user model.") detail_group.add_argument("-v", "--user_model_args", help="Arguments for initializing the user model.") # the retrieval system maintains ranking functions, accepts queries and # generates result lists, and in return receives user clicks to learn # from detail_group.add_argument("-s", "--system", help="Which system to use (e.g., pairwise, listwise).") detail_group.add_argument("-a", "--system_args", help="Arguments for " "the system (comparison method, learning " "algorithm and parameters...).") detail_group.add_argument("-o", "--output_dir", help="(Empty) directory for storing output generated by this" " experiment. Subdirectory for different folds will be generated" "automatically.") detail_group.add_argument("--output_dir_overwrite", default="False") detail_group.add_argument("-p", "--output_prefix", help="Prefix to be added to output filenames, e.g., the name of " "the data set, fold, etc. Output files will be stored as " "OUTPUT_DIR/PREFIX-RUN_ID.txt.gz") detail_group.add_argument("-e", "--experimenter", help="Experimenter type.") # run the parser if args_str: args = parser.parse_known_args(args_str.split())[0] else: args = parser.parse_known_args()[0] # determine whether to use config file or detailed args self.experiment_args = None if args.file: config_file = open(args.file) self.experiment_args = yaml.load(config_file) config_file.close() # overwrite with command-line options if given for arg, value in vars(args).items(): if value: self.experiment_args[arg] = value else: self.experiment_args = vars(args) # workaround - check if we have all the arguments needed if not ("training_queries" in self.experiment_args and "test_queries" in self.experiment_args and "feature_count" in self.experiment_args and "num_runs" in self.experiment_args and "num_queries" in self.experiment_args and "user_model" in self.experiment_args and "user_model_args" in self.experiment_args and "system" in self.experiment_args and "system_args" in self.experiment_args and "output_dir" in self.experiment_args): parser.print_help() sys.exit("Missing required arguments, please check the program" " arguments or configuration file. %s" % self.experiment_args) # set default values for optional arguments if not "query_sampling_method" in self.experiment_args: self.experiment_args["query_sampling_method"] = "random" if not "output_dir_overwrite" in self.experiment_args: self.experiment_args["output_dir_overwrite"] = False if not "experimenter" in self.experiment_args: self.experiment_args["experimenter"] = "experiment.LearningExperiment" if not "evaluation" in self.experiment_args: self.experiment_args["evaluation"] = "evaluation.NdcgEval" if not "processes" in self.experiment_args: self.experiment_args["processes"] = 0 # locate or create directory for the current fold if not os.path.exists(self.experiment_args["output_dir"]): os.makedirs(self.experiment_args["output_dir"]) elif not(self.experiment_args["output_dir_overwrite"]) and \ os.listdir(self.experiment_args["output_dir"]): # make sure the output directory is empty raise Exception("Output dir %s is not an empty directory. " "Please use a different directory, or move contents out " "of the way." % self.experiment_args["output_dir"]) logging.basicConfig(format='%(asctime)s %(module)s: %(message)s', level=logging.INFO) logging.info("Arguments: %s" % self.experiment_args) for k, v in sorted(self.experiment_args.iteritems()): logging.info("\t%s: %s" % (k, v)) config_bk = os.path.join(self.experiment_args["output_dir"], "config_bk.yml") logging.info("Backing up configuration to: %s" % config_bk) config_bk_file = open(config_bk, "w") yaml.dump(self.experiment_args, config_bk_file, default_flow_style=False) config_bk_file.close() # load training and test queries training_file = self.experiment_args["training_queries"] test_file = self.experiment_args["test_queries"] self.feature_count = self.experiment_args["feature_count"] logging.info("Loading training data: %s " % training_file) self.training_queries = load_queries(training_file, self.feature_count) logging.info("... found %d queries." % self.training_queries.get_size()) logging.info("Loading test data: %s " % test_file) self.test_queries = load_queries(test_file, self.feature_count) logging.info("... found %d queries." % self.test_queries.get_size()) # initialize and run the experiment num_run times self.num_runs = self.experiment_args["num_runs"] self.output_dir = self.experiment_args["output_dir"] self.output_prefix = self.experiment_args["output_prefix"] self.experimenter = get_class(self.experiment_args["experimenter"])
import sys import include import evaluation, query, ranker import numpy as np from ranker.AbstractRankingFunction import AbstractRankingFunction evaluation = evaluation.NdcgEval() bm25ranker = AbstractRankingFunction(["ranker.model.BM25"], 'first', 3, sample="utils.sample_fixed") queries = query.load_queries(sys.argv[1], 64) #print evaluation.evaluate_all(bm25ranker, queries) fh = open(sys.argv[1] + ".out.missing-b0.45.txt", "w") for k1 in sorted([2.6, 2.5]): for b in sorted([0.45]): #for k1 in np.arange(19.5, 100, 0.5): # for b in np.arange(-1, 1.2, 0.1): #for k3 in np.arange(100*itt, 100*(itt+1), 10): k3 = 0.0 bm25ranker.update_weights(np.array([k1,k3,b])) print >> fh, "k1:%f k3:%f b:%f score:%f" % (k1, k3, b, evaluation.evaluate_all(bm25ranker, queries)) fh.close()
def __init__(self, data_path, features): self.bi = comparison.BalancedInterleave() self.user_model = environment.CascadeUserModel('--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') self.queries = query.load_queries(data_path, features)
for m in range(3): print('Method ' + str(m)) if not os.path.exists('results/method' + str(m)): os.makedirs('results/method' + str(m)) test_scores = [] for i in range(folds): fold = i + 1 print('Fold ' + str(fold)) print('Creating model') model = LambdaRankHW( 64, 2, 'results/' + 'method' + str(m) + '/results' + str(fold) + '.csv') print('Model created') print('Loading queries') train_queries = query.load_queries( 'HP2003/Fold' + str(fold) + '/train.txt', 64) val_queries = query.load_queries( 'HP2003/Fold' + str(fold) + '/vali.txt', 64) test_queries = query.load_queries( 'HP2003/Fold' + str(fold) + '/test.txt', 64) print('Loaded\n') test_scores.append( model.train_with_queries(train_queries, val_queries, test_queries, epochs=15)) print('\n') method_scores.append(np.mean(test_scores)) print('Final scores: ') print(method_scores)
# Experiment 1 # Get optimum configuration for REM (number of k, and d) import sys, random import include import environment, evaluation, query, retrieval_system import time import datetime # init data, query_samples, d's train_queries = query.load_queries('../../DATA/NP2004/Fold1/train.txt', 64) test_queries = query.load_queries('../../DATA/NP2004/Fold1/test.txt', 64) query_samples = 5000 # how many queries we sample d_array = [2,3,4,5,6] k_array = [1,2,5,10] # init user model, evaluation methods user_model = environment.CascadeUserModel('--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') evaluation = evaluation.NdcgEval() # calculate using the lower dimensional slice(s) in d_array rem_ndcg_result = [[0 for i in range(len(d_array))] for j in range(len(k_array))] for n in range(0, len(k_array)): # number of evaluation for each k # make sure the number of total evaluation are equal for all k number_of_evaluation = query_samples / k_array[n] k = k_array[n] for idx in range(0,len(d_array)):
os.chdir("..") os.chdir("..") os.chdir("..") #feature_count=136 rankerDict = queryRankers() #feature_count=245 feature_count = 64 learner = retrieval_system.ListwiseLearningSystem( feature_count, '-w random -c comparison.ProbabilisticInterleave -r ranker.ProbabilisticRankingFunction -s 3 -d 0.1 -a 0.01' ) user_model = environment.CascadeUserModel( '--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') evaluation = evaluation.NdcgEval() training_queries = query.load_queries(sys.argv[1], feature_count) query_freq = {} for train in training_queries: if (len(train.__labels__) in query_freq): query_freq[len( train.__labels__)] = query_freq[len(train.__labels__)] + 1 else: query_freq[len(train.__labels__)] = 1 print query_freq test_queries = query.load_queries(sys.argv[2], feature_count) for i in range(20): q = training_queries[random.choice(training_queries.keys())] l = learner.get_ranked_list(q) c = user_model.get_clicks(l, q.get_labels()) s = learner.update_solution(c)
import sys import include import evaluation, query, ranker import numpy as np from ranker.AbstractRankingFunction import AbstractRankingFunction evaluation = evaluation.NdcgEval() bm25ranker = AbstractRankingFunction(["ranker.model.BM25"], 'first', 3, sample="utils.sample_fixed") queries = query.load_queries(sys.argv[1], 64) #print evaluation.evaluate_all(bm25ranker, queries) fh = open(sys.argv[1] + ".out.missing-b0.45.txt", "w") for k1 in sorted([2.6, 2.5]): for b in sorted([0.45]): #for k1 in np.arange(19.5, 100, 0.5): # for b in np.arange(-1, 1.2, 0.1): #for k3 in np.arange(100*itt, 100*(itt+1), 10): k3 = 0.0 bm25ranker.update_weights(np.array([k1, k3, b])) print >> fh, "k1:%f k3:%f b:%f score:%f" % ( k1, k3, b, evaluation.evaluate_all(bm25ranker, queries)) fh.close()
import sys, random import include import environment, evaluation, query, retrieval_system # init data, query_samples, d's queries = query.load_queries('../../data/NP2004/Fold1/test.txt', 64) query_samples = 10 # how many queries we sample d_array = [3,4,5,6] # init user model, evaluation methods user_model = environment.CascadeUserModel('--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') evaluation = evaluation.NdcgEval() # calculate using the full 64 dimensions full_learner = retrieval_system.ListwiseLearningSystem(64,'-w random -c comparison.ProbabilisticInterleave -r ranker.ProbabilisticRankingFunction -s 3 -d 0.1 -a 0.01') full_ndcg_result = [] for i in range(0,query_samples): q = queries[random.choice(queries.keys())] l = full_learner.get_ranked_list(q) c = user_model.get_clicks(l, q.get_labels()) s = full_learner.update_solution(c) full_ndcg_result.append( evaluation.evaluate_all(s, queries) ) full_ranker = full_learner.get_solution() # calculate using the lower dimensional slice(s) in d_array rem_ndcg_result = [[0 for i in range(len(d_array))] for j in range(query_samples)] rem_ranker = [] for idx in range(0,len(d_array)): d = d_array[idx] rem_learner = retrieval_system.ListwiseLearningSystemREMBO(64,d,'-w random -c comparison.ProbabilisticInterleave -r ranker.ProbabilisticRankingFunctionREMBO -s 3 -d 0.1 -a 0.01') for i in range(0,query_samples):
#!/usr/bin/python3 import json, os.path, requests, sys import config token = config.load_token('/vagrant/token/twitter.json') # load queries import query from query import i, q, count queries = query.load_queries('/vagrant/data/twitter_search_queries.json') query.init_queries_output_file(queries, '/vagrant/data') # execute each query for query in queries: path = '/vagrant/data/' + query[i] + '.json' # load previously saved tweets from the same query with open(path, 'r+') as tweets_file: tweets = json.load(tweets_file) # request url = 'https://api.twitter.com/1.1/search/tweets.json' params = { 'q': query[q], 'result_type': 'recent', 'count': count, 'since_id': tweets['since_id'] } headers = {'Authorization': 'Bearer ' + token} r = requests.get(url, params=params, headers=headers)
import query from query import i, q, count # path access_token_path = './credentials/access_token.json' # load access token access_token = auth.load_access_token(access_token_path) # paths queries_dir = './tweets' queries_file = 'queries.json' # load queries queries = query.load_queries(queries_dir + '/' + queries_file) query.init_queries_output_file(queries, queries_dir) # execute each query for query in queries: path = queries_dir + '/' + query[i] + '.json' # load previously saved tweets from the same query with open(path, 'r+') as tweets_file: tweets = json.load(tweets_file) # request url = 'https://api.twitter.com/1.1/search/tweets.json' params = { 'q': query[q], 'result_type': 'recent',
# Experiment 2 # Compare REM with original DBGD import sys, random import include import environment, evaluation, query, retrieval_system import time import datetime import numpy as np # init data, query_samples, d's train_queries = query.load_queries('../../DATA/NP2004/Fold1/train.txt', 64) test_queries = query.load_queries('../../DATA/NP2004/Fold1/test.txt', 64) query_samples = 5000 # how many queries we sample d = 3 k = 10 number_of_evaluation = query_samples / k # init user model, evaluation methods user_model = environment.CascadeUserModel( '--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') evaluation = evaluation.NdcgEval() rem_ndcg_evaluation_train = [] full_ndcg_evaluation_train = [] rem_ndcg_evaluation_test = [] full_ndcg_evaluation_test = [] for m in range(0, k): # for each k, we have different A matrix
# Stores the results for the current fold store_res.append(res) return store_res, ranker #test_queries = query.load_queries(os.path.normpath('./HP2003/Fold%d/test.txt' % fold), num_features) ## Run if __name__ == '__main__': print('Run Experiment.ipynb to see the plots and statistical tests') n_epochs = 200 measure_type = POINTWISE num_features = 64 num_folds = 5 results, best_ranker = experiment(n_epochs, measure_type, num_features, num_folds) print('Loading all test queries') test_queries = {} for fold in range(1, 6): test_queries = { **test_queries, **query.load_queries( os.path.normpath('./HP2003/Fold%d/test.txt' % fold), num_features) } print('Running test mndcg') test_results = report_test_ndcg(ranker, test_queries) print('Test mNDCG: %s' % sum(list(test_results.values())) / len(test_queries))
import retrieval_system, environment, evaluation, query import os from queryRankers import * import pickle os.chdir("..") os.chdir("..") os.chdir("..") #feature_count=136 rankerDict=queryRankers() #feature_count=245 feature_count=64 learner = retrieval_system.ListwiseLearningSystem(feature_count, '-w random -c comparison.ProbabilisticInterleave -r ranker.ProbabilisticRankingFunction -s 3 -d 0.1 -a 0.01') user_model = environment.CascadeUserModel('--p_click 0:0.0,1:1 --p_stop 0:0.0,1:0.0') evaluation = evaluation.NdcgEval() training_queries = query.load_queries(sys.argv[1], feature_count) query_freq={} for train in training_queries: if(len(train.__labels__) in query_freq): query_freq[len(train.__labels__)]=query_freq[len(train.__labels__)]+1 else: query_freq[len(train.__labels__)]=1 print query_freq test_queries = query.load_queries(sys.argv[2], feature_count) for i in range(20): q = training_queries[random.choice(training_queries.keys())] l = learner.get_ranked_list(q) c = user_model.get_clicks(l, q.get_labels()) s = learner.update_solution(c) print evaluation.evaluate_all(s, test_queries)
def compareSystemsHist(vali_queries,classifierPath,basic_ranker_path,clust_data_path,data,click): print "-Loading Data-" clf = pickle.load( open( classifierPath ) ) basic_ranker=pickle.load( open( basic_ranker_path ) ) clusterData=pickle.load(open(clust_data_path)) queryData=pickle.load(open(data)) ranker_tie="random" feature_count=basic_ranker.feature_count ranker_args="3" arg_str="" sample_send="sample_unit_sphere" iterations=100 rankers=[0]*2 rankers[0]=basic_ranker user_model = environment.CascadeUserModel(click) training_queries = query.load_queries(vali_queries, feature_count) compar_interleave=ProbabilisticInterleave(None) print "-Calculating-" ii=0 results=[] for qid in queryData.query_ranker.keys(): print str(float(ii)*100/float(len(queryData.query_ranker.keys())))+"%" ii+=1 q=training_queries.get_query(qid) for val in queryData.query_ranker[qid]: test=val #test=queryData.query_ranker[q][0] testWeights=str(test.tolist()) testWeights=testWeights.replace("[", "") testWeights=testWeights.replace("]", "") #weights = np.array([float(num) for num in testWeights.split(",")]) #print len(weights) ranker_tie="random" ranker_args="3" sample_send="sample_unit_sphere" rankers[1]=rankerClass.ProbabilisticRankingFunction(ranker_args, ranker_tie, feature_count, sample=sample_send, init=testWeights) second_win=0 for i in range(iterations): #q = training_queries.get_query(random.choice(training_queries.keys())) l, a = compar_interleave.interleave(rankers[0], rankers[1], q, 10) c = user_model.get_clicks(l, q.get_labels()) o = compar_interleave.infer_outcome(l, a, c, q) if(o>0): second_win+=1 elif(o==0): coin=random.random() if(coin>0.5): second_win+=1 result_com=float(second_win)/float(iterations) results.append(result_com) g=P.hist(results, bins = 20,range=[0,1]) P.xlabel("The win rate of the ranker",fontsize=20) P.ylabel("Number of rankers",fontsize=20) P.show(g)