def evaluate_ranking(sess, model, test_q_b_name, dbn, run_to_rerank, mql, mdl,
padding_value, fold_index, test_qrels_file, epoch,
run_name):
rel_docs_by_qry = {}
sim_scores_by_qry = {}
for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names,
q_name) in tqdm(
enumerate(
du.compute_test_fd_w2v_v2(test_q_b_name, dbn,
run_to_rerank, mql, mdl,
padding_value))):
if len(test_q_b_name[q_name]) == 0:
continue
feed_dict = {
model.X1: data1_test,
model.X1_len: data1_len_test,
model.X2: data2_test,
model.X2_len: data2_len_test,
model.training: False
}
pred = model.test_step(sess, feed_dict)
pred = np.array([p[0] for p in pred])
rel_docs_by_qry[q_name] = (
np.array(d_names)[np.argsort(-pred)])[0:1000]
sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000]
map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch),
rel_docs_by_qry,
sim_scores_by_qry,
test_qrels_file,
run_name,
output_folder='results')
return map_v
def true_rerank(sess, model, test_q_b_name, dbn, run_to_rerank, mql, mdl, padding_value, fold_index, test_qrels_file,
epoch, run_name, fasttext_vec_model, iwi, ii_dict, wn2v_model, idf_scores, output_f='../results'):
rel_docs_by_qry = {}
sim_scores_by_qry = {}
all_preds = {}
dnames_to_rerank_by_qry = {}
for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names, q_name, coeffs) in tqdm(enumerate(
compute_test_fd_w_coeffs(test_q_b_name, dbn, fasttext_vec_model, iwi, ii_dict, mql, mdl, padding_value,
run_to_rerank, wn2v_model, idf_scores))):
if len(test_q_b_name[q_name]) == 0:
continue
feed_dict = {model.X1: data1_test, model.X1_len: data1_len_test, model.X2: data2_test,
model.X2_len: data2_len_test, model.training: False, model.coeffs: coeffs}
# print('computing prediction %d of %d' % (i, len(test_q_b_name)))
pred = model.test_step(sess, feed_dict)
pred = np.array([p[0] for p in pred])
if q_name not in dnames_to_rerank_by_qry.keys():
dnames_to_rerank_by_qry[q_name] = []
all_preds[q_name] = []
dnames_to_rerank_by_qry[q_name].extend(d_names)
all_preds[q_name].extend(pred / np.max(pred))
for q_name in test_q_b_name.keys():
if q_name not in all_preds.keys():
continue
pred = np.array(all_preds[q_name])
rel_docs_by_qry[q_name] = (np.array(dnames_to_rerank_by_qry[q_name])[np.argsort(-pred)])[0:1000]
sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000]
map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch) + '_pure_neural', rel_docs_by_qry,
sim_scores_by_qry, test_qrels_file, run_name, output_folder=output_f)
print('map of pure neural model=%2.5f' % map_v)
return map_v
def eval_stochastic(sess,
model,
test_q_b_name,
dbn,
run_to_rerank,
mql,
mdl,
padding_value,
fold_index,
test_qrels_file,
epoch,
n_stoch_iter=50):
rel_docs_by_qry = {}
sim_scores_by_qry = {}
for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names,
q_name) in enumerate(
du.compute_test_fd_w2v_v2(test_q_b_name, dbn, run_to_rerank,
mql, mdl, padding_value)):
feed_dict = {
model.X1: data1_test,
model.X1_len: data1_len_test,
model.X2: data2_test,
model.X2_len: data2_len_test,
model.training: False
}
stoch_preds = np.zeros(len(d_names))
for _ in range(n_stoch_iter):
pred = model.test_step(sess, feed_dict)
pred = np.array(pred)
pred = np.reshape(pred, (-1))
stoch_preds += pred
stoch_preds /= n_stoch_iter
pred = stoch_preds
rel_docs_by_qry[q_name] = (
np.array(d_names)[np.argsort(-pred)])[0:1000]
sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000]
map_v = util.create_evaluate_ranking(
str(fold_index) + '_' + str(epoch), rel_docs_by_qry, sim_scores_by_qry,
test_qrels_file, PROG_NAME)
return map_v
def true_rerank_w_idf(sess, model, test_q_b_name, dbn, run_to_rerank, mql, mdl,
padding_value, fold_index, test_qrels_file, epoch,
run_name, iwi, idf_scores):
rel_docs_by_qry = {}
sim_scores_by_qry = {}
rel_scores_by_qry = get_relevance_scores_by_qry(run_to_rerank)
# normalize relevance scores
for qname, rel_scores_by_doc in rel_scores_by_qry.items():
max_score = max(rel_scores_by_doc.values())
for dname in rel_scores_by_doc.keys():
rel_scores_by_doc[dname] = rel_scores_by_doc[dname] / max_score
rel_scores_by_qry[qname] = rel_scores_by_doc
all_preds = {}
dnames_to_rerank_by_qry = {}
for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names,
q_name, bidf_coeffs) in tqdm(
enumerate(
du.compute_test_fd_tfidf(test_q_b_name, dbn, run_to_rerank,
mql, mdl, padding_value, iwi,
idf_scores))):
if len(test_q_b_name[q_name]) == 0:
continue
feed_dict = {
model.X1: data1_test,
model.X1_len: data1_len_test,
model.X2: data2_test,
model.X2_len: data2_len_test,
model.training: False,
model.idf_coeffs: bidf_coeffs
}
pred = model.test_step(sess, feed_dict)
pred = np.array([p[0] for p in pred])
dnames_to_rerank_by_qry[q_name] = d_names
all_preds[q_name] = pred / np.max(pred)
rel_docs_by_qry[q_name] = (
np.array(d_names)[np.argsort(-pred)])[0:1000]
sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000]
map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch) +
'_pure_neural',
rel_docs_by_qry,
sim_scores_by_qry,
test_qrels_file,
run_name,
output_folder='results')
print('map of pure neural model=%2.4f' % map_v)
for q_name in all_preds.keys():
d_names = dnames_to_rerank_by_qry[q_name]
rel_scores = np.zeros(len(d_names))
q_len = len(test_q_b_name[q_name])
for i in range(len(d_names)):
if len(test_q_b_name[q_name]) > 0:
qw_in_doc_cnt_normalized = sum([
1 for qw in test_q_b_name[q_name] if qw in dbn[d_names[i]]
])
qw_in_doc_cnt_normalized /= q_len
else:
qw_in_doc_cnt_normalized = 0
alpha = qw_in_doc_cnt_normalized
rel_scores[i] = (1 - alpha) * all_preds[q_name][
i] + alpha * rel_scores_by_qry[q_name][d_names[i]]
pred = np.array(rel_scores)
rel_docs_by_qry[q_name] = (
np.array(d_names)[np.argsort(-pred)])[0:1000]
sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000]
map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch),
rel_docs_by_qry,
sim_scores_by_qry,
test_qrels_file,
run_name,
output_folder='results')
return map_v
def eval_stochastic(sess,
model,
test_q_b_name,
dbn,
run_to_rerank,
mql,
mdl,
padding_value,
fold_index,
test_qrels_file,
epoch,
run_name,
n_stoch_iter=50):
print('evaluating...')
rel_docs_by_qry = {}
sim_scores_by_qry = {}
preds_bn = {}
for i, (data1_test, data2_test, data1_len_test, data2_len_test, d_names,
q_name) in tqdm(
enumerate(
du.compute_test_fd_w2v_v2(test_q_b_name, dbn,
run_to_rerank, mql, mdl,
padding_value))):
if len(test_q_b_name[q_name]) == 0:
continue
feed_dict = {
model.X1: data1_test,
model.X1_len: data1_len_test,
model.X2: data2_test,
model.X2_len: data2_len_test,
model.training: True
}
stoch_preds = []
for k in range(len(d_names)):
stoch_preds.append([])
for niter in range(n_stoch_iter):
pred = model.test_step(sess, feed_dict)
pred = np.array(pred)
pred = np.reshape(pred, (-1))
for j in range(len(pred)):
stoch_preds[j].append(pred[j])
stoch_preds = np.array(stoch_preds)
means = np.mean(stoch_preds, axis=1)
variances = np.var(stoch_preds, axis=1)
pred = means - variances
preds_bn[q_name] = (means, variances, d_names)
rel_docs_by_qry[q_name] = (
np.array(d_names)[np.argsort(-pred)])[0:1000]
sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000]
best_map = 0
best_alpha = 0
for alpha in np.arange(0, 2, 0.1):
for q_name in preds_bn.keys():
means, variances, d_names = preds_bn[q_name]
pred = means - (alpha * variances)
rel_docs_by_qry[q_name] = (
np.array(d_names)[np.argsort(-pred)])[0:1000]
sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000]
map_v = util.create_evaluate_ranking('fold=' + str(fold_index) +
'_epoch=' + str(epoch) +
'_alpha=%2.3f' + str(alpha),
rel_docs_by_qry,
sim_scores_by_qry,
test_qrels_file,
run_name,
output_folder='results')
if map_v > best_map:
best_map = map_v
best_alpha = alpha
print('alpha=%2.2f, map=%2.5f' % (best_alpha, best_map))
alpha = best_alpha
for q_name in preds_bn.keys():
means, variances, d_names = preds_bn[q_name]
pred = means - (alpha * variances)
rel_docs_by_qry[q_name] = (
np.array(d_names)[np.argsort(-pred)])[0:1000]
sim_scores_by_qry[q_name] = pred[np.argsort(-pred)][0:1000]
map_v = util.create_evaluate_ranking(str(fold_index) + '_' + str(epoch),
rel_docs_by_qry,
sim_scores_by_qry,
test_qrels_file,
run_name,
output_folder='results')
return map_v