"""

Update CBOW negative sampling model by training on a single sentence.

The sentence is a list of Vocab objects (or None, where the corresponding

word is not in the vocabulary. Called internally from `Word2Vec.train()`.

"""

for pos, word in enumerate(sentence):

if word is None:

continue # OOV word in the input sentence => skip

if model.reduce > 0:

reduced_window = random.randint(model.window) # `b` in the original word2vec code

else:

reduced_window = 0

# Combine all surrounding words into an appropriate input

start = max(0, pos - model.window + reduced_window)

l1 = matutils.zeros_aligned((model.layer1_size), dtype=REAL)#Initialize input

count = 0

for pos2, word2 in enumerate(sentence[start : pos + model.window + 1 - reduced_window], start):

if pos2 == pos or word2 is None:

pass

else:

l1 = l1 + model.syn0[word2.index]

count += 1

if count > 0:

l1 = l1 / count

neu1e = matutils.zeros_aligned((model.layer1_size), dtype=REAL)

for d in xrange(model.neg_samples+1):

if d == 0:

target_index = word.index

label = 1

else:

random_integer = random.randint(model.table_size)

target_index = model.table[random_integer]

if target_index == word.index:

continue

label = 0

l2a = model.syn1neg[target_index]

fa = 1.0 / (1.0 + exp(-dot(l1, l2a))) # propagate hidden -> output

ga = (label - fa) * alpha # vector of error gradients multiplied by the learning rate

neu1e += dot(ga,l2a)

model.syn1neg[target_index] += dot(ga, l1) # learn hidden -> output

for pos2, word2 in enumerate(sentence[start : pos + model.window + 1 - reduced_window], start):

if pos2 == pos or word2 is None:

pass

else:

model.syn0[word2.index] += neu1e

"""A single vocabulary item, used internally for constructing binary trees (incl. both word leaves and inner nodes)."""

def __init__(self, **kwargs):

self.count = 0

self.__dict__.update(kwargs)

def __lt__(self, other): # used for sorting in a priority queue

return self.count < other.count

def __str__(self):

vals = ['%s:%r' % (key, self.__dict__[key]) for key in sorted(self.__dict__) if not key.startswith('_')]

"""

Class for training, using and evaluating neural networks described in https://code.google.com/p/word2vec/

The model can be stored/loaded via its `save()` and `load()` methods, or stored/loaded in a format

compatible with the original word2vec implementation via `save_word2vec_format()` and `load_word2vec_format()`.

"""

def __init__(self, sentences=None, size=100, neg_samples = 5, alpha=0.025, window=5, min_count=5, max_count=inf, seed=1, workers=1, min_alpha=0.0001, reduce=1, alpha_decay=1.0, direction=0, auto_train=1):

"""

Initialize the model from an iterable of `sentences`. Each sentence is a

list of words (utf8 strings) that will be used for training.

See :class:`BrownCorpus` in this module for an example.

If you don't supply `sentences`, the model is left uninitialized -- use if

you plan to initialize it in some other way.

`size` is the dimensionality of the feature vectors.

`window` is the maximum distance between the current and predicted word within a sentence.

`alpha` is the initial learning rate (will linearly drop to zero as training progresses).

`seed` = for the random number generator.

`min_count` = ignore all words with total frequency lower than this.

`workers` = use this many worker threads to train the model (=faster training with multicore machines)

"""

self.vocab = {} # mapping from a word (string) to a Vocab object

self.index2word = [] # map from a word's matrix index (int) to word (string)

self.layer1_size = int(size)

self.neg_samples = int(neg_samples)

self.alpha = float(alpha)

self.window = int(window)

self.seed = seed

self.min_count = min_count

self.max_count = max_count

self.workers = workers

self.min_alpha = min_alpha

self.random_numbers = zeros(1000*self.neg_samples,dtype = uint32)

self.reduce = int(reduce)

self.alpha_decay = float(alpha_decay)

self.direction = int(direction) # direction is only implemented in Cython, not in the pure python/numpy version

self.auto_train = int(auto_train)

if sentences is not None:

self.build_vocab(sentences)

self.build_table()

if self.auto_train > 0:

self.train(sentences)

def create_binary_tree(self):

"""

Create a binary Huffman tree using stored vocabulary word counts. Frequent words

will have shorter binary codes. Called internally from `build_vocab()`.

"""

logger.info("constructing a huffman tree from %i words" % len(self.vocab))

# build the huffman tree

heap = self.vocab.values()

heapq.heapify(heap)

for i in xrange(len(self.vocab) - 1):

min1, min2 = heapq.heappop(heap), heapq.heappop(heap)

heapq.heappush(heap, Vocab(count=min1.count + min2.count, index=i + len(self.vocab), left=min1, right=min2))

# recurse over the tree, assigning a binary code to each vocabulary word

if heap:

max_depth, stack = 0, [(heap[0], [], [])]

while stack:

node, codes, points = stack.pop()

if node.index < len(self.vocab):

# leaf node => store its path from the root

node.code, node.point = codes, points

max_depth = max(len(codes), max_depth)

else:

# inner node => continue recursion

points = array(list(points) + [node.index - len(self.vocab)], dtype=uint32)

stack.append((node.left, array(list(codes) + [0], dtype=uint8), points))

stack.append((node.right, array(list(codes) + [1], dtype=uint8), points))

logger.info("built huffman tree with maximum node depth %i" % max_depth)

def build_vocab(self, sentences):

"""

Build vocabulary from a sequence of sentences (can be a once-only generator stream).

Each sentence must be a list of utf8 strings.

"""

logger.info("collecting all words and their counts")

sentence_no, vocab = -1, {}

total_words = lambda: sum(v.count for v in vocab.itervalues())

for sentence_no, sentence in enumerate(sentences):

if sentence_no % 10000 == 0:

logger.info("PROGRESS: at sentence #%i, processed %i words and %i word types" %

(sentence_no, total_words(), len(vocab)))

for word in sentence:

if word in vocab:

vocab[word].count += 1

else:

vocab[word] = Vocab(count=1)

logger.info("collected %i word types from a corpus of %i words and %i sentences" %

(len(vocab), total_words(), sentence_no + 1))

# assign a unique index to each word

self.vocab, self.index2word = {}, []

for word, v in vocab.iteritems():

if v.count >= self.min_count and v.count <= self.max_count:

try:

utils.to_utf8(word)

v.index = len(self.vocab)

self.index2word.append(word)

self.vocab[word] = v

self.vocab[word].count_power = pow(v.count, self.c_power)

self.vocab[word].count_power_2 = pow(v.count, 2*self.c_power)

except:

pass

logger.info("total %i word types after removing those with count<%s" % (len(self.vocab), self.min_count))

# add info about each word's Huffman encoding

self.create_binary_tree()

self.reset_weights()

def build_table(self):

power = 0.75

self.table_size = 100000000

table_size_real = self.table_size + 0.

vocab_size = len(self.vocab)

self.table = zeros((self.table_size),dtype = uint32)

train_words_pow = 0

for i in xrange(vocab_size):

cur_word = self.index2word[i]

train_words_pow += pow(self.vocab[cur_word].count, power)

train_words_pow += 0. #just to make sure this is represented as a real number

i = 0

cur_word = self.index2word[i]

d1 = pow(self.vocab[cur_word].count,power) / train_words_pow

for a in xrange(self.table_size):

self.table[a] = i

if a / table_size_real > d1:

i += 1

cur_word = self.index2word[i]

d1 += pow(self.vocab[cur_word].count,power) / train_words_pow

if i >= vocab_size:

i = vocab_size - 1

def train(self, sentences, total_words=None, word_count=0, chunksize=100):

"""

Update the model's neural weights from a sequence of sentences (can be a once-only generator stream).

Each sentence must be a list of utf8 strings.

"""

if FAST_VERSION < 0:

import warnings

warnings.warn("Cython compilation failed, training will be slow. Do you have Cython installed? `pip install cython`")

logger.info("training model with %i workers on %i vocabulary and %i features" % (self.workers, len(self.vocab), self.layer1_size))

if not self.vocab:

raise RuntimeError("you must first build vocabulary before training the model")

start, next_report = time.time(), [1.0]

word_count, total_words = [word_count], total_words or sum(v.count for v in self.vocab.itervalues())

jobs = Queue(maxsize=2 * self.workers) # buffer ahead only a limited number of jobs.. this is the reason we can't simply use ThreadPool :(

lock = threading.Lock() # for shared state (=number of words trained so far, log reports...)

def worker_train():

"""Train the model, lifting lists of sentences from the jobs queue."""

work = matutils.zeros_aligned(self.layer1_size, dtype=REAL) # each thread must have its own work memory

neu1 = matutils.zeros_aligned(self.layer1_size, dtype=REAL) # each thread must have its own work memory

while True:

job = jobs.get()

if job is None: # data finished, exit

break

# update the learning rate before every job

alpha = max(self.min_alpha, self.alpha * (1 - 1.0 * self.alpha_decay * word_count[0] / total_words))

# how many words did we train on? out-of-vocabulary (unknown) words do not count

job_words = sum(train_sentence(self, sentence, alpha, work, neu1) for sentence in job)

with lock:

word_count[0] += job_words

elapsed = time.time() - start

if elapsed >= next_report[0]:

logger.info("PROGRESS: at %.2f%% words, alpha %.05f, %.0f words/s" %

(100.0 * word_count[0] / total_words, alpha, word_count[0] / elapsed if elapsed else 0.0))

next_report[0] = elapsed + 1.0 # don't flood the log, wait at least a second between progress reports

workers = [threading.Thread(target=worker_train) for _ in xrange(self.workers)]

for thread in workers:

thread.daemon = True # make interrupting the process with ctrl+c easier

thread.start()

# convert input strings to Vocab objects (or None for OOV words), and start filling the jobs queue

no_oov = ([self.vocab.get(word, None) for word in sentence] for sentence in sentences)

for job_no, job in enumerate(utils.grouper(no_oov, chunksize)):

logger.debug("putting job #%i in the queue, qsize=%i" % (job_no, jobs.qsize()))

jobs.put(job)

logger.info("reached the end of input; waiting to finish %i outstanding jobs" % jobs.qsize())

for _ in xrange(self.workers):

jobs.put(None) # give the workers heads up that they can finish -- no more work!

for thread in workers:

thread.join()

elapsed = time.time() - start

logger.info("training on %i words took %.1fs, %.0f words/s" %

(word_count[0], elapsed, word_count[0] / elapsed if elapsed else 0.0))

return word_count[0]

def reset_weights(self):

"""Reset all projection weights to an initial (untrained) state, but keep the existing vocabulary."""

random.seed(self.seed)

self.syn0 = matutils.zeros_aligned((len(self.vocab), self.layer1_size), dtype=REAL)

self.syn1neg = matutils.zeros_aligned((len(self.vocab), self.layer1_size), dtype=REAL)

self.syn0 += (random.rand(len(self.vocab), self.layer1_size) - 0.5) / self.layer1_size

self.syn0norm = None

def save_word2vec_format(self, fname, binary=False):

"""

Store the input-hidden weight matrix in the same format used by the original

C word2vec-tool, for compatibility.

"""

logger.info("storing %sx%s projection weights into %s" % (len(self.vocab), self.layer1_size, fname))

assert (len(self.vocab), self.layer1_size) == self.syn0.shape

with open(fname, 'wb') as fout:

fout.write("%s %s\n" % self.syn0.shape)

# store in sorted order: most frequent words at the top

for word, vocab in sorted(self.vocab.iteritems(), key=lambda item: -item[1].count):

word = utils.to_utf8(word) # always store in utf8

row = self.syn0[vocab.index]

if binary:

fout.write("%s %s\n" % (word, row.tostring()))

else:

fout.write("%s %s\n" % (word, ' '.join("%f" % val for val in row)))

@classmethod

def load_word2vec_format(cls, fname, binary=False):

"""

Load the input-hidden weight matrix from the original C word2vec-tool format.

Note that the information loaded is incomplete (the binary tree is missing),

so while you can query for word similarity etc., you cannot continue training

with a model loaded this way.

"""

logger.info("loading projection weights from %s" % (fname))

with open(fname) as fin:

header = fin.readline()

vocab_size, layer1_size = map(int, header.split()) # throws for invalid file format

result = Word2Vec(size=layer1_size)

result.syn0 = empty((vocab_size, layer1_size), dtype=REAL)

if binary:

binary_len = dtype(REAL).itemsize * layer1_size

for line_no in xrange(vocab_size):

# mixed text and binary: read text first, then binary

word = []

while True:

ch = fin.read(1)

if ch == ' ':

word = ''.join(word)

break

word.append(ch)

result.vocab[word] = Vocab(index=line_no, count=vocab_size - line_no)

result.index2word.append(word)

result.syn0[line_no] = fromstring(fin.read(binary_len), dtype=REAL)

fin.read(1) # newline

else:

for line_no, line in enumerate(fin):

parts = line.split()

assert len(parts) == layer1_size + 1

word, weights = parts[0], map(REAL, parts[1:])

result.vocab[word] = Vocab(index=line_no, count=vocab_size - line_no)

result.index2word.append(word)

result.syn0[line_no] = weights

logger.info("loaded %s matrix from %s" % (result.syn0.shape, fname))

result.init_sims()

return result

def most_similar(self, positive=[], negative=[], topn=10):

"""

Find the top-N most similar words. Positive words contribute positively towards the

similarity, negative words negatively.

This method computes cosine similarity between a simple mean of the projection

weight vectors of the given words, and corresponds to the `word-analogy` and

`distance` scripts in the original word2vec implementation.

Example::

>>> trained_model.most_similar(positive=['woman', 'king'], negative=['man'])

[('queen', 0.50882536), ...]

"""

self.init_sims()

if isinstance(positive, basestring) and not negative:

# allow calls like most_similar('dog'), as a shorthand for most_similar(['dog'])

positive = [positive]

# add weights for each word, if not already present; default to 1.0 for positive and -1.0 for negative words

positive = [(word, 1.0) if isinstance(word, basestring) else word for word in positive]

negative = [(word, -1.0) if isinstance(word, basestring) else word for word in negative]

# compute the weighted average of all words

all_words, mean = set(), []

for word, weight in positive + negative:

if word in self.vocab:

mean.append(weight * matutils.unitvec(self.syn0[self.vocab[word].index]))

all_words.add(self.vocab[word].index)

else:

raise KeyError("word '%s' not in vocabulary" % word)

if not mean:

raise ValueError("cannot compute similarity with no input")

mean = matutils.unitvec(array(mean).mean(axis=0)).astype(REAL)

dists = dot(self.syn0norm, mean)

if not topn:

return dists

best = argsort(dists)[::-1][:topn + len(all_words)]

# ignore (don't return) words from the input

result = [(self.index2word[sim], dists[sim]) for sim in best if sim not in all_words]

return result[:topn]

def doesnt_match(self, words):

"""

Which word from the given list doesn't go with the others?

Example::

>>> trained_model.doesnt_match("breakfast cereal dinner lunch".split())

'cereal'

"""

words = [word for word in words if word in self.vocab] # filter out OOV words

logger.debug("using words %s" % words)

if not words:

raise ValueError("cannot select a word from an empty list")

vectors = vstack(matutils.unitvec(self.syn0[self.vocab[word].index]) for word in words).astype(REAL)

mean = matutils.unitvec(vectors.mean(axis=0)).astype(REAL)

dists = dot(vectors, mean)

return sorted(zip(dists, words))[0][1]

def __getitem__(self, word):

"""

Return a word's representations in vector space, as a 1D numpy array.

Example::

>>> trained_model['woman']

array([ -1.40128313e-02, ...]

"""

return self.syn0[self.vocab[word].index]

def __contains__(self, word):

return word in self.vocab

def similarity(self, w1, w2):

"""

Compute cosine similarity between two words.

Example::

>>> trained_model.similarity('woman', 'man')

0.73723527

>>> trained_model.similarity('woman', 'woman')

1.0

"""

return dot(matutils.unitvec(self[w1]), matutils.unitvec(self[w2]))

def init_sims(self):

if getattr(self, 'syn0norm', None) is None:

logger.info("precomputing L2-norms of word weight vectors")

self.syn0norm = vstack(matutils.unitvec(vec) for vec in self.syn0).astype(REAL)

def accuracy(self, questions, restrict_vocab=30000):

"""

Compute accuracy of the model. `questions` is a filename where lines are

4-tuples of words, split into sections by ": SECTION NAME" lines.

See https://code.google.com/p/word2vec/source/browse/trunk/questions-words.txt for an example.

The accuracy is reported (=printed to log and returned as a list) for each

section separately, plus there's one aggregate summary at the end.

Use `restrict_vocab` to ignore all questions containing a word whose frequency

is not in the top-N most frequent words (default top 30,000).

This method corresponds to the `compute-accuracy` script of the original C word2vec.

"""

ok_vocab = dict(sorted(self.vocab.iteritems(), key=lambda item: -item[1].count)[:restrict_vocab])

ok_index = set(v.index for v in ok_vocab.itervalues())

def log_accuracy(section):

correct, incorrect = section['correct'], section['incorrect']

if correct + incorrect > 0:

logger.info("%s: %.1f%% (%i/%i)" %

(section['section'], 100.0 * correct / (correct + incorrect),

correct, correct + incorrect))

sections, section = [], None

for line_no, line in enumerate(open(questions)):

# TODO: use level3 BLAS (=evaluate multiple questions at once), for speed

if line.startswith(': '):

# a new section starts => store the old section

if section:

sections.append(section)

log_accuracy(section)

section = {'section': line.lstrip(': ').strip(), 'correct': 0, 'incorrect': 0}

else:

if not section:

raise ValueError("missing section header before line #%i in %s" % (line_no, questions))

try:

a, b, c, expected = [word.lower() for word in line.split()] # TODO assumes vocabulary preprocessing uses lowercase, too...

except:

logger.info("skipping invalid line #%i in %s" % (line_no, questions))

if a not in ok_vocab or b not in ok_vocab or c not in ok_vocab or expected not in ok_vocab:

logger.debug("skipping line #%i with OOV words: %s" % (line_no, line))

continue

ignore = set(self.vocab[v].index for v in [a, b, c]) # indexes of words to ignore

predicted = None

# find the most likely prediction, ignoring OOV words and input words

for index in argsort(self.most_similar(positive=[b, c], negative=[a], topn=False))[::-1]:

if index in ok_index and index not in ignore:

predicted = self.index2word[index]

if predicted != expected:

logger.debug("%s: expected %s, predicted %s" % (line.strip(), expected, predicted))

break

section['correct' if predicted == expected else 'incorrect'] += 1

if section:

# store the last section, too

sections.append(section)

log_accuracy(section)

total = {'section': 'total', 'correct': sum(s['correct'] for s in sections), 'incorrect': sum(s['incorrect'] for s in sections)}

log_accuracy(total)

sections.append(total)

return sections

def __str__(self):

"""Iterate over sentences from the Brown corpus (part of NLTK data)."""

def __init__(self, dirname):

self.dirname = dirname

def __iter__(self):

for fname in os.listdir(self.dirname):

fname = os.path.join(self.dirname, fname)

if not os.path.isfile(fname):

continue

for line in open(fname):

# each file line is a single sentence in the Brown corpus

# each token is WORD/POS_TAG

token_tags = [t.split('/') for t in line.split() if len(t.split('/')) == 2]

# ignore words with non-alphabetic tags like ",", "!" etc (punctuation, weird stuff)

words = ["%s/%s" % (token.lower(), tag[:2]) for token, tag in token_tags if tag[:2].isalpha()]

if not words: # don't bother sending out empty sentences

continue

"""Iterate over sentences from the "text8" corpus, unzipped from http://mattmahoney.net/dc/text8.zip ."""

def __init__(self, fname):

self.fname = fname

def __iter__(self):

# the entire corpus is one gigantic line -- there are no sentence marks at all

# so just split the sequence of tokens arbitrarily: 1 sentence = 1000 tokens

sentence, rest, max_sentence_length = [], '', 1000

with open(self.fname) as fin:

while True:

text = rest + fin.read(8192) # avoid loading the entire file (=1 line) into RAM

if text == rest: # EOF

sentence.extend(rest.split()) # return the last chunk of words, too (may be shorter/longer)

if sentence:

yield sentence

break

last_token = text.rfind(' ') # the last token may have been split in two... keep it for the next iteration

words, rest = (text[:last_token].split(), text[last_token:].strip()) if last_token >= 0 else ([], text)

sentence.extend(words)

while len(sentence) >= max_sentence_length:

yield sentence[:max_sentence_length]

"""Iterate over sentences from the Holmes training dataset ."""

def __init__(self, dir):

self.dir = dir

def __iter__(self):

sentence, rest, max_sentence_length = [], '', 1000

for i in sorted(os.listdir(self.dir)):

with open(os.path.join(self.dir,i)) as fin:

while True:

text = rest + fin.read(8192) # avoid loading the entire file (=1 line) into RAM

if text == rest: # EOF

sentence.extend(rest.split()) # return the last chunk of words, too (may be shorter/longer)

if sentence:

yield sentence

break

last_token = text.rfind(' ') # the last token may have been split in two... keep it for the next iteration

words, rest = (text[:last_token].split(), text[last_token:].strip()) if last_token >= 0 else ([], text)

sentence.extend(words)

while len(sentence) >= max_sentence_length:

yield sentence[:max_sentence_length]

def __init__(self, fname):

"""Simple format: one sentence = one line; words already preprocessed and separated by whitespace."""

self.fname = fname

def __iter__(self):

for line in open(self.fname):