def evaluate_ranking(sess, model, test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value, fold_index, test_qrels_file, epoch, run_name): rel_docs_by_qry = {} sim_scores_by_qry = {} for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names, q_name) in tqdm( enumerate( du.compute_test_fd_w2v_v2(test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value))): if len(test_q_b_name[q_name]) == 0: continue feed_dict = { model.X1: data1_test, model.X1_len: data1_len_test, model.X2: data2_test, model.X2_len: data2_len_test, model.training: False } pred = model.test_step(sess, feed_dict) pred = np.array([p[0] for p in pred]) rel_docs_by_qry[q_name] = ( np.array(d_names)[np.argsort(-pred)])[0:1000] sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000] map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch), rel_docs_by_qry, sim_scores_by_qry, test_qrels_file, run_name, output_folder='results') return map_v
def true_rerank(sess, model, test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value, fold_index, test_qrels_file, epoch, run_name, fasttext_vec_model, iwi, ii_dict, wn2v_model, idf_scores, output_f='../results'): rel_docs_by_qry = {} sim_scores_by_qry = {} all_preds = {} dnames_to_rerank_by_qry = {} for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names, q_name, coeffs) in tqdm(enumerate( compute_test_fd_w_coeffs(test_q_b_name, dbn, fasttext_vec_model, iwi, ii_dict, mql, mdl, padding_value, run_to_rerank, wn2v_model, idf_scores))): if len(test_q_b_name[q_name]) == 0: continue feed_dict = {model.X1: data1_test, model.X1_len: data1_len_test, model.X2: data2_test, model.X2_len: data2_len_test, model.training: False, model.coeffs: coeffs} # print('computing prediction %d of %d' % (i, len(test_q_b_name))) pred = model.test_step(sess, feed_dict) pred = np.array([p[0] for p in pred]) if q_name not in dnames_to_rerank_by_qry.keys(): dnames_to_rerank_by_qry[q_name] = [] all_preds[q_name] = [] dnames_to_rerank_by_qry[q_name].extend(d_names) all_preds[q_name].extend(pred / np.max(pred)) for q_name in test_q_b_name.keys(): if q_name not in all_preds.keys(): continue pred = np.array(all_preds[q_name]) rel_docs_by_qry[q_name] = (np.array(dnames_to_rerank_by_qry[q_name])[np.argsort(-pred)])[0:1000] sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000] map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch) + '_pure_neural', rel_docs_by_qry, sim_scores_by_qry, test_qrels_file, run_name, output_folder=output_f) print('map of pure neural model=%2.5f' % map_v) return map_v
def eval_stochastic(sess, model, test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value, fold_index, test_qrels_file, epoch, n_stoch_iter=50): rel_docs_by_qry = {} sim_scores_by_qry = {} for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names, q_name) in enumerate( du.compute_test_fd_w2v_v2(test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value)): feed_dict = { model.X1: data1_test, model.X1_len: data1_len_test, model.X2: data2_test, model.X2_len: data2_len_test, model.training: False } stoch_preds = np.zeros(len(d_names)) for _ in range(n_stoch_iter): pred = model.test_step(sess, feed_dict) pred = np.array(pred) pred = np.reshape(pred, (-1)) stoch_preds += pred stoch_preds /= n_stoch_iter pred = stoch_preds rel_docs_by_qry[q_name] = ( np.array(d_names)[np.argsort(-pred)])[0:1000] sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000] map_v = util.create_evaluate_ranking( str(fold_index) + '_' + str(epoch), rel_docs_by_qry, sim_scores_by_qry, test_qrels_file, PROG_NAME) return map_v
def true_rerank_w_idf(sess, model, test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value, fold_index, test_qrels_file, epoch, run_name, iwi, idf_scores): rel_docs_by_qry = {} sim_scores_by_qry = {} rel_scores_by_qry = get_relevance_scores_by_qry(run_to_rerank) # normalize relevance scores for qname, rel_scores_by_doc in rel_scores_by_qry.items(): max_score = max(rel_scores_by_doc.values()) for dname in rel_scores_by_doc.keys(): rel_scores_by_doc[dname] = rel_scores_by_doc[dname] / max_score rel_scores_by_qry[qname] = rel_scores_by_doc all_preds = {} dnames_to_rerank_by_qry = {} for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names, q_name, bidf_coeffs) in tqdm( enumerate( du.compute_test_fd_tfidf(test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value, iwi, idf_scores))): if len(test_q_b_name[q_name]) == 0: continue feed_dict = { model.X1: data1_test, model.X1_len: data1_len_test, model.X2: data2_test, model.X2_len: data2_len_test, model.training: False, model.idf_coeffs: bidf_coeffs } pred = model.test_step(sess, feed_dict) pred = np.array([p[0] for p in pred]) dnames_to_rerank_by_qry[q_name] = d_names all_preds[q_name] = pred / np.max(pred) rel_docs_by_qry[q_name] = ( np.array(d_names)[np.argsort(-pred)])[0:1000] sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000] map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch) + '_pure_neural', rel_docs_by_qry, sim_scores_by_qry, test_qrels_file, run_name, output_folder='results') print('map of pure neural model=%2.4f' % map_v) for q_name in all_preds.keys(): d_names = dnames_to_rerank_by_qry[q_name] rel_scores = np.zeros(len(d_names)) q_len = len(test_q_b_name[q_name]) for i in range(len(d_names)): if len(test_q_b_name[q_name]) > 0: qw_in_doc_cnt_normalized = sum([ 1 for qw in test_q_b_name[q_name] if qw in dbn[d_names[i]] ]) qw_in_doc_cnt_normalized /= q_len else: qw_in_doc_cnt_normalized = 0 alpha = qw_in_doc_cnt_normalized rel_scores[i] = (1 - alpha) * all_preds[q_name][ i] + alpha * rel_scores_by_qry[q_name][d_names[i]] pred = np.array(rel_scores) rel_docs_by_qry[q_name] = ( np.array(d_names)[np.argsort(-pred)])[0:1000] sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000] map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch), rel_docs_by_qry, sim_scores_by_qry, test_qrels_file, run_name, output_folder='results') return map_v
def eval_stochastic(sess, model, test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value, fold_index, test_qrels_file, epoch, run_name, n_stoch_iter=50): print('evaluating...') rel_docs_by_qry = {} sim_scores_by_qry = {} preds_bn = {} for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names, q_name) in tqdm( enumerate( du.compute_test_fd_w2v_v2(test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value))): if len(test_q_b_name[q_name]) == 0: continue feed_dict = { model.X1: data1_test, model.X1_len: data1_len_test, model.X2: data2_test, model.X2_len: data2_len_test, model.training: True } stoch_preds = [] for k in range(len(d_names)): stoch_preds.append([]) for niter in range(n_stoch_iter): pred = model.test_step(sess, feed_dict) pred = np.array(pred) pred = np.reshape(pred, (-1)) for j in range(len(pred)): stoch_preds[j].append(pred[j]) stoch_preds = np.array(stoch_preds) means = np.mean(stoch_preds, axis=1) variances = np.var(stoch_preds, axis=1) pred = means - variances preds_bn[q_name] = (means, variances, d_names) rel_docs_by_qry[q_name] = ( np.array(d_names)[np.argsort(-pred)])[0:1000] sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000] best_map = 0 best_alpha = 0 for alpha in np.arange(0, 2, 0.1): for q_name in preds_bn.keys(): means, variances, d_names = preds_bn[q_name] pred = means - (alpha * variances) rel_docs_by_qry[q_name] = ( np.array(d_names)[np.argsort(-pred)])[0:1000] sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000] map_v = util.create_evaluate_ranking('fold=' + str(fold_index) + '_epoch=' + str(epoch) + '_alpha=%2.3f' + str(alpha), rel_docs_by_qry, sim_scores_by_qry, test_qrels_file, run_name, output_folder='results') if map_v > best_map: best_map = map_v best_alpha = alpha print('alpha=%2.2f, map=%2.5f' % (best_alpha, best_map)) alpha = best_alpha for q_name in preds_bn.keys(): means, variances, d_names = preds_bn[q_name] pred = means - (alpha * variances) rel_docs_by_qry[q_name] = ( np.array(d_names)[np.argsort(-pred)])[0:1000] sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000] map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch), rel_docs_by_qry, sim_scores_by_qry, test_qrels_file, run_name, output_folder='results') return map_v