def query(self, query, k=None, indices=None, sort=True, return_scores=False):
models = self.retrieval_models
weights = maxabs_scale(self.weights) # max 1 does not crash [0,1]
agg_fn = self.aggregation_fn
# It's important that all retrieval model return the same number of documents.
all_scores = [m.query(query, k=k, indices=indices, sort=False, return_scores=True)[1] for m in models]
if weights is not None:
all_scores = [weight * scores for weight, scores in zip(all_scores, weights)]
scores = np.vstack(all_scores)
if callable(agg_fn):
aggregated_scores = agg_fn(scores)
else:
numpy_fn = getattr(np, agg_fn)
aggregated_scores = numpy_fn(scores, axis=0)
# combined = aggregate_dicts(combined, agg_fn=agg_fn, sort=True)
# only cut-off at k if this is the final (sorted) output
ind = argtopk(aggregated_scores, k) if sort else np.arange(aggregated_scores.shape[0])
if return_scores:
return ind, aggregated_scores[ind]
else:
return ind
def query(self, query, k=None, indices=None):
if self._fit_X is None:
raise NotFittedError
q = super().transform([query])
if indices is not None:
fit_X = self._fit_X[indices]
else:
fit_X = self._fit_X
# both fit_X and q are l2-normalized
D = linear_kernel(q, fit_X)
ind = argtopk(D[0], k)
return ind
def query(self, query):
E, m = self.embedding, self.m
V = self.vocabulary
priors = self.priors
q = self.analyzer(query)
posterior = eqe1(E, q, priors)
expansion = V[argtopk(posterior, m)]
expanded_query = " ".join(q + expansion)
if self.verbose > 0:
print("[eqlm] Expanded query: '{}'".format(expanded_query))
# employ retrieval model
return self.retrieval_model.query(expanded_query)
def query(self, query):
E, m = self.embedding, self.m
V = self.vocabulary
priors = self.priors
q = self.analyzer(query)
posterior = eqe1(E, q, priors)
expansion = V[argtopk(posterior, m)]
expanded_query = " ".join(q + expansion)
if self.verbose > 0:
print("[eqlm] Expanded query: '{}'".format(expanded_query))
# employ retrieval model
return self.retrieval_model.query(expanded_query)
def query(self, query, k=None, indices=None, return_scores=False, sort=True):
if self._fit_X is None:
raise NotFittedError
q = super().transform([query])
if indices is not None:
fit_X = self._fit_X[indices]
else:
fit_X = self._fit_X
# both fit_X and q are l2-normalized
D = linear_kernel(q, fit_X)
ind = argtopk(D[0], k) if sort else np.arange(D.shape[1])
if return_scores:
return ind, D[0,ind]
else:
return ind
def expand(posterior, vocabulary, m=10):
"""
>>> vocabulary = "a b c".split()
>>> posterior = [0.9, 0.1, 0.42]
>>> expand(posterior, vocabulary, 0)
[]
>>> expand(posterior, vocabulary, 1)
['a']
>>> expand(posterior, vocabulary, 2)
['a', 'c']
"""
if m <= 0:
return []
vocabulary = np.asarray(vocabulary)
expansions = vocabulary[argtopk(posterior, m)]
return list(expansions)
def expand(posterior, vocabulary, m=10):
"""
>>> vocabulary = "a b c".split()
>>> posterior = [0.9, 0.1, 0.42]
>>> expand(posterior, vocabulary, 0)
[]
>>> expand(posterior, vocabulary, 1)
['a']
>>> expand(posterior, vocabulary, 2)
['a', 'c']
"""
if m <= 0:
return []
vocabulary = np.asarray(vocabulary)
expansions = vocabulary[argtopk(posterior, m)]
return list(expansions)
def evaluate_results(qids_rs, Y, k):
values = defaultdict(list)
for qid, r in qids_rs:
gold = harvest(Y, qid)
gold_topk = gold[argtopk(gold, k)]
R = np.count_nonzero(gold_topk)
# real ndcg
idcg = rm.dcg_at_k(gold_topk, k)
ndcg = rm.dcg_at_k(r, k) / idcg
values["ndcg"].append(ndcg)
# Verified
# MAP@k
ap = rm.average_precision(r)
values["MAP"].append(ap)
# MRR - compute by hand
ind = np.asarray(r).nonzero()[0]
mrr = (1. / (ind[0] + 1)) if ind.size else 0.
values["MRR"].append(mrr)
# R precision
# R = min(R, k) # ok lets be fair.. you cant get more than k
# we dont need that anymore, since we chop of the remainder
# before computing R
recall = rm.recall(r, R)
values["recall"].append(recall)
# precision = rm.precision_at_k(pad(scored_result, k), k)
precision = rm.precision(r)
values["precision"].append(precision)
f1 = f1_score(precision, recall)
values["f1_score"].append(f1)
# Safe variant does not fail if len(r) < k
p_at_5 = rm.safe_precision_at_k(r, 5)
values["precision@5"].append(p_at_5)
p_at_10 = rm.safe_precision_at_k(r, 10)
values["precision@10"].append(p_at_10)
return values
def evaluate_results(qids_rs, Y, k):
values = defaultdict(list)
for qid, r in qids_rs:
gold = harvest(Y, qid)
gold_topk = gold[argtopk(gold, k)]
R = np.count_nonzero(gold_topk)
# real ndcg
idcg = rm.dcg_at_k(gold_topk, k)
ndcg = rm.dcg_at_k(r, k) / idcg
values["ndcg"].append(ndcg)
# Verified
# MAP@k
ap = rm.average_precision(r)
values["MAP"].append(ap)
# MRR - compute by hand
ind = np.asarray(r).nonzero()[0]
mrr = (1. / (ind[0] + 1)) if ind.size else 0.
values["MRR"].append(mrr)
# R precision
# R = min(R, k) # ok lets be fair.. you cant get more than k
# we dont need that anymore, since we chop of the remainder
# before computing R
recall = rm.recall(r, R)
values["recall"].append(recall)
# precision = rm.precision_at_k(pad(scored_result, k), k)
precision = rm.precision(r)
values["precision"].append(precision)
f1 = f1_score(precision, recall)
values["f1_score"].append(f1)
# Safe variant does not fail if len(r) < k
p_at_5 = rm.safe_precision_at_k(r, 5)
values["precision@5"].append(p_at_5)
p_at_10 = rm.safe_precision_at_k(r, 10)
values["precision@10"].append(p_at_10)
return values
def query(self,
query,
k=None,
indices=None,
return_scores=False,
sort=True):
if self._fit_X is None:
raise NotFittedError
q = super().transform([query])
if indices is not None:
fit_X = self._fit_X[indices]
else:
fit_X = self._fit_X
# both fit_X and q are l2-normalized
D = linear_kernel(q, fit_X)
ind = argtopk(D[0], k) if sort else np.arange(D.shape[1])
if return_scores:
return ind, D[0, ind]
else:
return ind
def query(self,
query,
k=None,
indices=None,
sort=True,
return_scores=False):
models = self.retrieval_models
weights = maxabs_scale(self.weights) # max 1 does not crash [0,1]
agg_fn = self.aggregation_fn
# It's important that all retrieval model return the same number of documents.
all_scores = [
m.query(query,
k=k,
indices=indices,
sort=False,
return_scores=True)[1] for m in models
]
if weights is not None:
all_scores = [
weight * scores for weight, scores in zip(all_scores, weights)
]
scores = np.vstack(all_scores)
if callable(agg_fn):
aggregated_scores = agg_fn(scores)
else:
numpy_fn = getattr(np, agg_fn)
aggregated_scores = numpy_fn(scores, axis=0)
# combined = aggregate_dicts(combined, agg_fn=agg_fn, sort=True)
# only cut-off at k if this is the final (sorted) output
ind = argtopk(aggregated_scores, k) if sort else np.arange(
aggregated_scores.shape[0])
if return_scores:
return ind, aggregated_scores[ind]
else:
return ind
def evaluate(self, X, Y, k=20, verbose=0, replacement=0, n_jobs=1):
"""
:X: [(qid, str)] query id, query string pairs
:Y: pandas dataseries with qid,docid index or [dict]
:k: Limit the result for all metrics to this value, the models are also
given a hint of how many they should return.
:replacement: 0 means that (query, doc) pairs not prevalent in Y will
not be considered relevant, None means that those are not considered
(skipped).
"""
# rs = []
# if n_jobs > 1:
# return process_and_evaluate(self, X, Y, k, n_jobs)
values = defaultdict(list)
for qid, query in X:
# execute query
if verbose > 0:
print(qid, ":", query)
t0 = timer()
# if replacement is None, we need to drop after querying
result = self.query(query, k=(None if replacement is None else k))
values["time_per_query"].append(timer() - t0)
# if verbose > 0:
# print(result[:k])
# result = result[:k] # TRIM HERE
# soak the generator
scored_result = [
harvest(Y, qid, docid, replacement) for docid in result
]
if replacement is None:
scored_result, notfound = filter_none(scored_result)
values["gold_not_found"].append(notfound)
if k is not None:
# dont let the models cheat by returning more than k
r = scored_result[:k]
else:
# if k is None, consider all
r = scored_result
# if verbose > 0:
# print(r)
# gold = np.array(list(Y[qid].values()))
gold = harvest(Y, qid)
import sys
# print(gold, file=sys.stderr)
topk_indices = argtopk(gold, k)
print(topk_indices, file=sys.stderr)
gold_topk = gold[topk_indices]
# print('Top k in gold standard:', gold_topk, file=sys.stderr)
R = np.count_nonzero(gold_topk)
if verbose > 0:
print("Retrieved {} relevant out of {} possible.".format(
np.count_nonzero(r), R))
# real ndcg
idcg = rm.dcg_at_k(gold_topk, k)
ndcg = rm.dcg_at_k(scored_result, k) / idcg
values["ndcg"].append(ndcg)
# Verified
# MAP@k
ap = rm.average_precision(r)
values["MAP"].append(ap)
# MRR - compute by hand
ind = np.asarray(r).nonzero()[0]
mrr = (1. / (ind[0] + 1)) if ind.size else 0.
values["MRR"].append(mrr)
# R precision
# R = min(R, k) # ok lets be fair.. you cant get more than k
# we dont need that anymore, since we chop of the remainder
# before computing R
recall = rm.recall(r, R)
values["recall"].append(recall)
# precision = rm.precision_at_k(pad(scored_result, k), k)
precision = rm.precision(r)
values["precision"].append(precision)
f1 = f1_score(precision, recall)
values["f1_score"].append(f1)
# Safe variant does not fail if len(r) < k
p_at_5 = rm.safe_precision_at_k(r, 5)
values["precision@5"].append(p_at_5)
p_at_10 = rm.safe_precision_at_k(r, 10)
values["precision@10"].append(p_at_10)
# rs.append(r)
if verbose > 0:
# print("Precision: {:.4f}".format(precision))
# print("Recall: {:.4f}".format(recall))
# print("F1-Score: {:.4f}".format(f1))
print("AP: {:.4f}".format(ap))
print("RR: {:.4f}".format(mrr))
print("NDCG: {:.4f}".format(ndcg))
return values
def evaluate(self, X, Y, k=20, verbose=0, replacement=0, n_jobs=1):
"""
:X: [(qid, str)] query id, query string pairs
:Y: pandas dataseries with qid,docid index or [dict]
:k: Limit the result for all metrics to this value, the models are also
given a hint of how many they should return.
:replacement: 0 means that (query, doc) pairs not prevalent in Y will
not be considered relevant, None means that those are not considered
(skipped).
"""
# rs = []
# if n_jobs > 1:
# return process_and_evaluate(self, X, Y, k, n_jobs)
values = defaultdict(list)
for qid, query in X:
# execute query
if verbose > 0:
print(qid, ":", query)
t0 = timer()
# if replacement is None, we need to drop after querying
result = self.query(query, k=(None if replacement is None else k))
values["time_per_query"].append(timer() - t0)
# if verbose > 0:
# print(result[:k])
# result = result[:k] # TRIM HERE
# soak the generator
scored_result = [harvest(Y, qid, docid, replacement)
for docid in result]
if replacement is None:
scored_result, notfound = filter_none(scored_result)
values["gold_not_found"].append(notfound)
if k is not None:
# dont let the models cheat by returning more than k
r = scored_result[:k]
else:
# if k is None, consider all
r = scored_result
# if verbose > 0:
# print(r)
# gold = np.array(list(Y[qid].values()))
gold = harvest(Y, qid)
import sys
# print(gold, file=sys.stderr)
topk_indices = argtopk(gold, k)
print(topk_indices, file=sys.stderr)
gold_topk = gold[topk_indices]
# print('Top k in gold standard:', gold_topk, file=sys.stderr)
R = np.count_nonzero(gold_topk)
if verbose > 0:
print("Retrieved {} relevant out of {} possible."
.format(np.count_nonzero(r), R))
# real ndcg
idcg = rm.dcg_at_k(gold_topk, k)
ndcg = rm.dcg_at_k(scored_result, k) / idcg
values["ndcg"].append(ndcg)
# Verified
# MAP@k
ap = rm.average_precision(r)
values["MAP"].append(ap)
# MRR - compute by hand
ind = np.asarray(r).nonzero()[0]
mrr = (1. / (ind[0] + 1)) if ind.size else 0.
values["MRR"].append(mrr)
# R precision
# R = min(R, k) # ok lets be fair.. you cant get more than k
# we dont need that anymore, since we chop of the remainder
# before computing R
recall = rm.recall(r, R)
values["recall"].append(recall)
# precision = rm.precision_at_k(pad(scored_result, k), k)
precision = rm.precision(r)
values["precision"].append(precision)
f1 = f1_score(precision, recall)
values["f1_score"].append(f1)
# Safe variant does not fail if len(r) < k
p_at_5 = rm.safe_precision_at_k(r, 5)
values["precision@5"].append(p_at_5)
p_at_10 = rm.safe_precision_at_k(r, 10)
values["precision@10"].append(p_at_10)
# rs.append(r)
if verbose > 0:
# print("Precision: {:.4f}".format(precision))
# print("Recall: {:.4f}".format(recall))
# print("F1-Score: {:.4f}".format(f1))
print("AP: {:.4f}".format(ap))
print("RR: {:.4f}".format(mrr))
print("NDCG: {:.4f}".format(ndcg))
return values
