def reciprocal_rank(model, base, derived, max_neighbors=None, pos=None, verbose=False):
neighbors = get_neighbors(model.predict(base, verbose), model.space, max_neighbors, pos)
rank = 0
for i, (w, _) in enumerate(neighbors):
if w == derived:
rank = i + 1
break
if verbose:
print("%s: correct target '%s' is at rank %d out of %d" % (base, derived, rank, len(neighbors)))
reciprocal_rank = 0 if rank == 0 else 1 / float(rank)
return reciprocal_rank
def reciprocal_rank_scores(model, test_pairs, max_neighbors=None, pos=None, verbose=False):
scores = []
for (base, derived) in test_pairs:
neighbors = get_neighbors(model.predict(base, verbose), model.space, max_neighbors, pos)
rank = 0
for i, (w, _) in enumerate(neighbors):
if w == derived:
rank = i + 1
break
if verbose:
print("%s: correct target '%s' is at rank %d out of %d" % (base, derived, rank, len(neighbors)))
reciprocal_rank = 0 if rank == 0 else 1 / float(rank)
scores.append(reciprocal_rank)
return sp.array(scores)
def score(model, test_pairs, n_neighbors=5, pos=None, verbose=False):
hits = 0
# TODO: so speed up, pos-filter space here and only once for the complete pattern
for (base, derived) in test_pairs:
neighbors = get_neighbors(model.predict(base, verbose), model.space, n_neighbors, pos)
if verbose:
print base, "=>", derived
print neighbors
if derived in map(itemgetter(0), neighbors):
hits += 1
if verbose:
print "HIT!"
else:
if verbose:
print "MISS"
if verbose:
print
score = float(hits) / len(test_pairs)
if verbose:
print "=> Score: %d out of %d (%.2f%%)" % (hits, len(test_pairs), score)
return hits, score
