"""

Find the top-N most similar words. Positive words contribute positively towards the

similarity, negative words negatively.

`model.word_vectors` must be a matrix of word embeddings (already L2-normalized),

and its format must be either 2d numpy (dense) or scipy.sparse.csr.

"""

if isinstance(positive, basestring) and not negative:

# allow calls like most_similar('dog'), as a shorthand for most_similar(['dog'])

positive = [positive]

# add weights for each word, if not already present; default to 1.0 for positive and -1.0 for negative words

positive = [

(word, 1.0) if isinstance(word, (basestring, numpy.ndarray)) else word

for word in positive]

negative = [

(word, -1.0) if isinstance(word, (basestring, numpy.ndarray)) else word

for word in negative]

# compute the weighted average of all words

all_words, mean = set(), []

for word, weight in positive + negative:

if isinstance(word, numpy.ndarray):

mean.append(weight * word)

elif word in model.word2id:

word_index = model.word2id[word]

mean.append(weight * model.word_vectors[word_index])

all_words.add(word_index)

else:

raise KeyError("word '%s' not in vocabulary" % word)

if not mean:

raise ValueError("cannot compute similarity with no input")

if scipy.sparse.issparse(model.word_vectors):

mean = scipy.sparse.vstack(mean)

else:

mean = numpy.array(mean)

mean = matutils.unitvec(mean.mean(axis=0)).astype(model.word_vectors.dtype)

dists = model.word_vectors.dot(mean.T).flatten()

if not topn:

return dists

best = numpy.argsort(dists)[::-1][:topn + len(all_words)]

# ignore (don't return) words from the input

result = [(model.id2word[sim], float(dists[sim])) for sim in best if sim not in all_words]

correct, incorrect = section['correct'], section['incorrect']

if correct + incorrect > 0:

logger.info("%s: %.1f%% (%i/%i)" %

(section['section'], 100.0 * correct / (correct + incorrect),

"""

Compute accuracy of the word embeddings.

`questions` is a filename where lines are 4-tuples of words, split into

sections by ": SECTION NAME" lines.

See https://code.google.com/p/word2vec/source/browse/trunk/questions-words.txt for an example.

The accuracy is reported (=printed to log and returned as a list) for each

section separately, plus there's one aggregate summary at the end.

Only evaluate on words in `word2id` (such as 30k most common words), ignoring

any test examples where any of the four words falls outside `word2id`.

This method corresponds to the `compute-accuracy` script of the original C word2vec.

"""

if ok_words is None:

ok_words = model.word2id

sections, section = [], None

for line_no, line in enumerate(utils.smart_open(questions)):

line = utils.to_unicode(line)

if line.startswith(': '):

# a new section starts => store the old section

if section:

sections.append(section)

log_accuracy(section)

section = {'section': line.lstrip(': ').strip(), 'correct': 0, 'incorrect': 0}

else:

if not section:

raise ValueError("missing section header before line #%i in %s" % (line_no, questions))

try:

a, b, c, expected = [word.lower() for word in line.split()] # TODO assumes vocabulary preprocessing uses lowercase, too...

except:

logger.info("skipping invalid line #%i in %s" % (line_no, questions))

if a not in ok_words or b not in ok_words or c not in ok_words or expected not in ok_words:

logger.debug("skipping line #%i with OOV words: %s" % (line_no, line.strip()))

continue

ignore = set(model.word2id[v] for v in [a, b, c]) # indexes of words to ignore

predicted = None

# find the most likely prediction, ignoring OOV words and input words

sims = most_similar(model, positive=[b, c], negative=[a], topn=False)

for index in numpy.argsort(sims)[::-1]:

if model.id2word[index] in ok_words and index not in ignore:

predicted = model.id2word[index]

if predicted != expected:

logger.debug("%s: expected %s, predicted %s" % (line.strip(), expected, predicted))

break

section['correct' if predicted == expected else 'incorrect'] += 1

if section:

# store the last section, too

sections.append(section)

log_accuracy(section)

total = {'section': 'total', 'correct': sum(s['correct'] for s in sections), 'incorrect': sum(s['incorrect'] for s in sections)}

log_accuracy(total)

sections.append(total)

"""

Convert raw counts from `get_coccur` into positive PMI values (as per Levy & Goldberg),

in place.

The result is an efficient stream of sparse word vectors (=no extra data copy).

"""

logger.info("computing PPMI on co-occurence counts")

# following lines a bit tedious, as we try to avoid making temporary copies of the (large) `cooccur` matrix

marginal_word = cooccur.sum(axis=1)

marginal_context = cooccur.sum(axis=0)

cooccur /= marginal_word[:, None] # #(w, c) / #w

cooccur /= marginal_context # #(w, c) / (#w * #c)

cooccur *= marginal_word.sum() # #(w, c) * D / (#w * #c)

numpy.log(cooccur, out=cooccur) # PMI = log(#(w, c) * D / (#w * #c))

logger.info("shifting PMI scores by log(k) with k=%s" % (k_shift, ))

cooccur -= numpy.log(k_shift) # shifted PMI = log(#(w, c) * D / (#w * #c)) - log(k)

logger.info("clipping PMI scores to be non-negative PPMI")

cooccur.clip(0.0, out=cooccur) # SPPMI = max(0, log(#(w, c) * D / (#w * #c)) - log(k))

logger.info("normalizing PPMI word vectors to unit length")

for i, vec in enumerate(cooccur):

cooccur[i] = matutils.unitvec(vec)

def __init__(self, corpus):

# serialize PPMI vectors into an explicit sparse CSR matrix, in RAM, so we can do

# dot products more easily

def __init__(self, corpus, id2word, s_exponent=0.0):

logger.info("calculating truncated SVD")

lsi = gensim.models.LsiModel(corpus, id2word=id2word, num_topics=DIM, chunksize=1000)

self.singular_scaled = lsi.projection.s ** s_exponent

# embeddings = left singular vectors scaled by the (exponentiated) singular values