class Word2Vec: def __init__(self, input_file_name, output_file_name): self.min_count = 5 self.emb_dimension = 100 self.batch_size = 64 self.window_size = 5 self.iteration = 1 self.initial_lr = 0.001 self.data = InputData(input_file_name, self.min_count) self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension, self.batch_size, self.window_size, self.iteration, self.initial_lr, self.min_count) self.use_cuda = torch.cuda.is_available() if self.use_cuda: self.skip_gram_model.cuda() self.optimizer = optim.SGD( self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): """Multiple training. Returns: None. """ pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = Variable(torch.LongTensor(pos_u)) pos_v = Variable(torch.LongTensor(pos_v)) neg_v = Variable(torch.LongTensor(neg_v)) if self.use_cuda: pos_u = pos_u.cuda() pos_v = pos_v.cuda() neg_v = neg_v.cuda() self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() process_bar.set_description("Loss: %0.8f, lr: %0.6f" % (loss.data, self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr self.skip_gram_model.save_embedding( self.data.id2word, self.output_file_name, self.use_cuda)
class Word2Vec: def __init__(self, input_file_name, output_file_name, emb_dimension=100, batch_size=50, window_size=5, iteration=5, initial_lr=0.025, neg_num=5, min_count=5): self.data = InputData(input_file_name, min_count) self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.iteration = iteration self.initial_lr = initial_lr self.neg_num = neg_num self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension) self.skip_gram_model.cuda() self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) count = int(batch_count) // 3 for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, self.neg_num) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = Variable(torch.LongTensor(pos_u)).cuda() pos_v = Variable(torch.LongTensor(pos_v)).cuda() neg_v = Variable(torch.LongTensor(neg_v)).cuda() self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() process_bar.set_description("Loss: %0.8f, lr: %0.6f" % (loss.item(), self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr if i != 0 and i % count == 0: self.skip_gram_model.save_embedding(self.data.id2word,self.output_file_name + str(i)) self.skip_gram_model.save_embedding(self.data.id2word, self.output_file_name + 'final')
class Word2Vec: def __init__(self, input_file_name, output_file_name, emb_dimension=100, batch_size=50, window_size=5, iteration=1, initial_lr=0.025, min_count=1): """Initilize class parameters. Args: input_file_name: Name of a text data from file. Each line is a sentence splited with space. output_file_name: Name of the final embedding file. emb_dimention: Embedding dimention, typically from 50 to 500. batch_size: The count of word pairs for one forward. window_size: Max skip length between words. iteration: Control the multiple training iterations. initial_lr: Initial learning rate. min_count: The minimal word frequency, words with lower frequency will be filtered. Returns: None. """ self.data = InputData(input_file_name, min_count) self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.iteration = iteration self.initial_lr = initial_lr self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension) self.use_cuda = torch.cuda.is_available() if self.use_cuda: self.skip_gram_model.cuda() self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): """Multiple training. Returns: None. """ pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) # self.skip_gram_model.save_embedding( # self.data.id2word, 'begin_embedding.txt', self.use_cuda) for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = Variable(torch.LongTensor(pos_u)) pos_v = Variable(torch.LongTensor(pos_v)) neg_v = Variable(torch.LongTensor(neg_v)) if self.use_cuda: pos_u = pos_u.cuda() pos_v = pos_v.cuda() neg_v = neg_v.cuda() self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr self.skip_gram_model.save_embedding(self.data.id2word, self.output_file_name, self.use_cuda)
class Word2Vec: def __init__( self, input_file_name, input_wvectors, input_cvectors, input_ps, input_ns, output_file_name, emb_dimension=100, batch_size=50, window_size=5, kn=20, iteration=1, initial_lr=0.001, clip=1.0, min_count=30, batch_num_to_valid=100000, ): """Initilize class parameters. Args: input_file_name: Name of a text data from file. Each line is a sentence splited with space. input_vectors: Pretrained vector input_psns: Pretrained positive sample & negative sample output_file_name: Name of the final embedding file. emb_dimention: Embedding dimention, typically from 50 to 500. batch_size: The count of word pairs for one forward. window_size: Max skip length between words. kn: k neighbors. iteration: Control the multiple training iterations. initial_lr: Initial learning rate. min_count: The minimal word frequency, words with lower frequency will be filtered. Returns: None. """ self.data = InputData(input_file_name, min_count) self.pre_wvectors = InputVector(input_wvectors) self.pre_cvectors = InputVector(input_cvectors) self.ps_w = load_from_pkl(input_ps) self.ns_w = load_from_pkl(input_ns) self.ps = convert_word_to_id(self.ps_w, self.data.word2id) self.ns = convert_word_to_id(self.ns_w, self.data.word2id) self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.kn = kn self.iteration = iteration self.initial_lr = initial_lr self.clip = clip self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension, self.pre_wvectors, self.pre_cvectors) self.use_cuda = torch.cuda.is_available() if self.use_cuda: self.skip_gram_model.cuda() self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) self.batch_num_to_valid = batch_num_to_valid def train(self, similarity_test_paths, synset_paths, analogy_paths): """Multiple training. Returns: None. """ pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) # self.skip_gram_model.save_embedding( # self.data.id2word, 'begin_embedding.txt', self.use_cuda) best_scores = dict() tmp_emb_dir = os.path.join(tempfile.gettempdir(), 'embedding') tmp_emb_path = os.path.join( tmp_emb_dir, ''.join(random.sample(string.ascii_letters + string.digits, 16))) for epoch in range(self.iteration): for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) pos_u, mask_pos_u = self.data.get_ps_batch( pos_pairs, self.ps, self.kn) neg_u, mask_neg_u = self.data.get_ns_batch( pos_pairs, self.ns, self.kn) pair_u = [pair[0] for pair in pos_pairs] pair_v = [pair[1] for pair in pos_pairs] pair_u = Variable(torch.LongTensor(pair_u)) pair_v = Variable(torch.LongTensor(pair_v)) pos_u = Variable(torch.LongTensor(pos_u)) mask_pos_u = Variable(torch.FloatTensor(mask_pos_u)) neg_u = Variable(torch.LongTensor(neg_u)) mask_neg_u = Variable(torch.FloatTensor(mask_neg_u)) if self.use_cuda: pair_u = pair_u.cuda() pair_v = pair_v.cuda() pos_u = pos_u.cuda() mask_pos_u = mask_pos_u.cuda() neg_u = neg_u.cuda() mask_neg_u = mask_neg_u.cuda() self.optimizer.zero_grad() ''' param = self.skip_gram_model.parameters() tmp = [] try: while True: tmp.append(param.__next__()) except: pass ''' loss = self.skip_gram_model.forward(pair_u, pair_v, pos_u, mask_pos_u, neg_u, mask_neg_u) loss.backward() torch.nn.utils.clip_grad_norm( self.skip_gram_model.parameters(), self.clip) self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr if i % self.batch_num_to_valid == 0: logging.info('epoch%d_batch%d, evaluating...' % (epoch, i)) self.save_embedding(self.data.id2word, tmp_emb_path, self.use_cuda) best_scores, save_flag = evaluation( tmp_emb_path, similarity_test_paths, synset_paths, analogy_paths, best_scores) if save_flag == True: emb_save_path = self.output_file_name + "_epoch%d_batch%d" % ( epoch, i) shutil.move(tmp_emb_path, emb_save_path) logging.info('Save current embedding to %s' % emb_save_path) self.skip_gram_model.save_embedding(self.data.id2word, self.output_file_name, self.use_cuda) logging.info('final evaluating...') self.save_embedding(self.data.id2word, tmp_emb_path, self.use_cuda) best_scores, save_flag = evaluation(tmp_emb_path, similarity_test_paths, synset_paths, analogy_paths, best_scores) if save_flag == True: emb_save_path = self.output_file_name + "_epoch%d" % epoch shutil.move(tmp_emb_path, emb_save_path) logging.info('Save current embedding to %s' % emb_save_path) def save_embedding(self, id2word, file_name, use_cuda): """Save all embeddings to file. As this class only record word id, so the map from id to word has to be transfered from outside. Args: id2word: map from word id to word. file_name: file name. Returns: None. """ if use_cuda: embedding = self.skip_gram_model.u_embeddings.weight.cpu( ).data.numpy() else: embedding = self.skip_gram_model.u_embeddings.weight.data.numpy() fout = open(file_name, 'w') fout.write('%d %d\n' % (len(id2word), self.emb_dimension)) for wid, w in id2word.items(): e = embedding[wid] e = ' '.join(map(lambda x: str(x), e)) fout.write('%s %s\n' % (w, e))
class Word2Vec: def __init__(self, input_file_name, output_file_name, emb_dimension=100, batch_size=100, window_size=5, iteration=5, initial_lr=0.025, min_count=5, using_hs=False, using_neg=False, context_size=2, hidden_size=128, cbow=None, skip_gram=None): print("\nInput File loading......\n") self.data = InputData(input_file_name, min_count) print("\nInput File loaded.\n") self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.iteration = iteration self.initial_lr = initial_lr self.context_size = context_size self.hidden_size = hidden_size self.using_hs = using_hs self.using_neg = using_neg self.cbow = cbow self.skip_gram = skip_gram if self.skip_gram is not None and self.skip_gram: self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension) print("skip_gram_model", self.skip_gram_model) self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) if self.cbow is not None and self.cbow: self.cbow_model = CBOW(self.emb_size, self.emb_dimension) print("CBOW_model", self.cbow_model) self.optimizer = optim.SGD(self.cbow_model.parameters(), lr=self.initial_lr) def skip_gram_train(self): """Multiple training. Returns: None. """ print("Skip_Gram Training......") pair_count = self.data.evaluate_pair_count(self.window_size) print("pair_count", pair_count) batch_count = self.iteration * pair_count / self.batch_size print("batch_count", batch_count) process_bar = tqdm(range(int(batch_count))) self.skip_gram_model.save_embedding(self.data.id2word, 'skip_gram_begin_embedding.txt') for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) if self.using_hs: pos_pairs, neg_pairs = self.data.get_pairs_by_huffman( pos_pairs) else: pos_pairs, neg_pairs = self.data.get_pairs_by_neg_sampling( pos_pairs, 5) pos_u = [int(pair[0]) for pair in pos_pairs] pos_v = [int(pair[1]) for pair in pos_pairs] neg_u = [int(pair[0]) for pair in neg_pairs] neg_v = [int(pair[1]) for pair in neg_pairs] self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_u, neg_v) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) print("Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr print("Skip_Gram Trained and Saving File......") self.skip_gram_model.save_embedding(self.data.id2word, self.output_file_name) print("Skip_Gram Trained and Saved File.") def cbow_train(self): print("CBOW Training......") self.cbow_model.save_embedding(self.data.id2word, 'cbow_begin_embedding.txt') pos_all_pairs = self.data.get_cbow_batch_all_pairs( self.batch_size, self.context_size) pair_count = len(pos_all_pairs) process_bar = tqdm(range(int(pair_count / self.batch_size))) for _ in process_bar: pos_pairs = self.data.get_cbow_batch_pairs(self.batch_size, self.window_size) if self.using_hs: pos_pairs, neg_pairs = self.data.get_cbow_pairs_by_huffman( pos_pairs) else: pos_pairs, neg_pairs = self.data.get_cbow_pairs_by_neg_sampling( pos_pairs, self.context_size) pos_u = [pair[0] for pair in pos_pairs] pos_v = [int(pair[1]) for pair in pos_pairs] neg_u = [pair[0] for pair in neg_pairs] neg_v = [int(pair[1]) for pair in neg_pairs] self.optimizer.zero_grad() loss = self.cbow_model.forward(pos_u, pos_v, neg_u, neg_v) loss.backward() self.optimizer.step() print("CBOW Trained and Saving File......") self.cbow_model.save_embedding(self.data.id2word, self.output_file_name) print("CBOW Trained and Saved File.")
class Word2Vec: def __init__(self, input_file_name, output_file_name, emb_dimension=100, batch_size=100, window_size=5, iteration=5, initial_lr=0.025, min_count=5, using_hs=False, using_neg=False, context_size=2, hidden_size=128, cbow=None, skip_gram=None): """Initilize class parameters. Args: input_file_name: Name of a text data from file. Each line is a sentence splited with space. output_file_name: Name of the final embedding file. emb_dimention: Embedding dimention, typically from 50 to 500. batch_size: The count of word pairs for one forward. window_size: Max skip length between words. iteration: Control the multiple training iterations. initial_lr: Initial learning rate. min_count: The minimal word frequency, words with lower frequency will be filtered. using_hs: Whether using hierarchical softmax. Returns: None. """ print("\nInput File loading......\n") self.data = InputData(input_file_name, min_count) print("\nInput File loaded.\n") print("Input Data", self.data) self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) print("emb_size", self.emb_size) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.iteration = iteration self.initial_lr = initial_lr self.context_size = context_size self.hidden_size = hidden_size self.using_hs = using_hs self.using_neg = using_neg self.cbow = cbow self.skip_gram = skip_gram if self.skip_gram is not None and self.skip_gram: self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension) print("skip_gram_model", self.skip_gram_model) self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) if self.cbow is not None and self.cbow: # self.cbow_model = CBOW(self.emb_size, self.context_size, self.emb_dimension, self.hidden_size) self.cbow_model = CBOW(self.emb_size, self.emb_dimension) print("CBOW_model", self.cbow_model) self.optimizer = optim.SGD(self.cbow_model.parameters(), lr=self.initial_lr) # @profile def skip_gram_train(self): """Multiple training. Returns: None. """ pair_count = self.data.evaluate_pair_count(self.window_size) print("pair_count", pair_count) batch_count = self.iteration * pair_count / self.batch_size print("batch_count", batch_count) process_bar = tqdm(range(int(batch_count))) self.skip_gram_model.save_embedding(self.data.id2word, 'skip_gram_begin_embedding.txt') for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) if self.using_hs: pos_pairs, neg_pairs = self.data.get_pairs_by_huffman( pos_pairs) else: pos_pairs, neg_pairs = self.data.get_pairs_by_neg_sampling( pos_pairs, 5) pos_u = [int(pair[0]) for pair in pos_pairs] pos_v = [int(pair[1]) for pair in pos_pairs] neg_u = [int(pair[0]) for pair in neg_pairs] neg_v = [int(pair[1]) for pair in neg_pairs] self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_u, neg_v) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) print("Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr self.skip_gram_model.save_embedding(self.data.id2word, self.output_file_name) def cbow_train(self): print("CBOW Training......") pair_count = self.data.evaluate_pair_count(self.context_size * 2 + 1) print("pair_count", pair_count) batch_count = self.iteration * pair_count / self.batch_size print("batch_count", batch_count) process_bar = tqdm(range(int(batch_count))) self.cbow_model.save_embedding(self.data.id2word, 'cbow_begin_embedding.txt') for i in process_bar: pos_pairs = self.data.get_cbow_batch_all_pairs( self.batch_size, self.context_size) if self.using_hs: pos_pairs, neg_pairs = self.data.get_cbow_pairs_by_huffman( pos_pairs) else: pos_pairs, neg_pairs = self.data.get_cbow_pairs_by_neg_sampling( pos_pairs, self.context_size) pos_u = [pair[0] for pair in pos_pairs] pos_v = [int(pair[1]) for pair in pos_pairs] neg_u = [pair[0] for pair in neg_pairs] neg_v = [int(pair[1]) for pair in neg_pairs] self.optimizer.zero_grad() loss = self.cbow_model.forward(pos_u, pos_v, neg_u, neg_v) # loss = self.cbow_model.forwards(pos_v, pos_u, neg_v, neg_u) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) print("Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr print("CBOW Trained and Saving File......") self.cbow_model.save_embedding(self.data.id2word, self.output_file_name) print("CBOW Trained and Saved File.")
class Word2Vec: def __init__(self, output_file_name, output_sense_name, emb_dimension=128, K=5, batch_size=1, window_size=5, iteration=1, initial_lr=0.1, createClusterLambda=1.5, min_count=0): """Initilize class parameters. Args: input_file_name: Name of a text data from file. Each line is a sentence splited with space. output_file_name: Name of the final embedding file. emb_dimention: Embedding dimention, typically from 50 to 500. batch_size: The count of word pairs for one forward. window_size: Max skip length between words. iteration: Control the multiple training iterations. initial_lr: Initial learning rate. min_count: The minimal word frequency, words with lower frequency will be filtered. Returns: None. """ self.data = InputData(min_count) self.output_file_name = output_file_name self.output_sense_name = output_sense_name self.emb_size = len(self.data.node2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.K = K self.iteration = iteration self.initial_lr = initial_lr self.createClusterLambda = createClusterLambda self.skip_gram_model = SkipGramModel(self.emb_size, self.K, self.emb_dimension) self.use_cuda = torch.cuda.is_available() if self.use_cuda: self.skip_gram_model.cuda() self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): """Multiple training. Returns: None. """ pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) total_pos_pairs = self.data.get_node_pairs(self.window_size) print("training\n") for t in process_bar: pos_pairs = total_pos_pairs[t] neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] # right=[] cnt = 0 curword = pos_u[cnt] contextwords = [] contextwords_cuda = [] while cnt < len(pos_u): contextwords.append(pos_v[cnt]) contextwords_cuda.append(pos_v[cnt]) cnt += 1 contextembedding = torch.zeros(self.emb_dimension) contextwords_cuda = Variable(torch.LongTensor(contextwords_cuda)) if self.use_cuda: contextwords_cuda = contextwords_cuda.cuda() emb_v = self.skip_gram_model.v_embeddings(contextwords_cuda) if self.use_cuda: emb_v_data = emb_v.cpu().data else: emb_v_data = emb_v.data for i in range(len(contextwords)): contextembedding += emb_v_data[i] # torch.add(contextembedding,emb_v_data[i,:],out=emb_v_data_total) emb_v_data_avg = contextembedding / (len(contextwords)) # torch.div(emb_v_data_total,len(contextwords),out=emb_v_data_avg) minDist = np.inf rightsense = 0 mu = torch.Tensor(self.emb_dimension) if self.skip_gram_model.num_sense[curword] == self.K: nC = self.K else: nC = self.skip_gram_model.num_sense[curword] + 1 prob = torch.Tensor(nC) for k in range(self.skip_gram_model.num_sense[curword]): torch.div(self.skip_gram_model.clusterCenter[curword, k, :], self.skip_gram_model.clusterCount[curword][k], out=mu) x_norm = torch.norm(emb_v_data_avg, p=2) y_norm = torch.norm(mu, p=2) summ = 0 for p in range(self.emb_dimension): summ += emb_v_data_avg[p] * mu[p] dist = 1 - summ / (x_norm * y_norm) prob[k] = dist if dist < minDist: minDist = dist rightsense = k if self.skip_gram_model.num_sense[curword] < self.K: if self.createClusterLambda < minDist: prob[self.skip_gram_model. num_sense[curword]] = self.createClusterLambda rightsense = self.skip_gram_model.num_sense[curword] self.skip_gram_model.num_sense[curword] += 1 for i in range(self.emb_dimension): self.skip_gram_model.clusterCenter[curword][rightsense][ i] += emb_v_data_avg[i] self.skip_gram_model.clusterCount[curword][rightsense] += 1 # for i in range(len(contextwords)): # right.append(rightsense) self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v, rightsense, self.use_cuda) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) if t * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * t / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr self.skip_gram_model.save_embedding(self.data.id2node, self.output_file_name, self.output_sense_name, self.use_cuda)
class Word2Vec: def __init__(self, wikidump_filename, output_text_filename, emb_dimension, batch_size, window_size, iteration, initial_lr, min_count): self.data = InputData(wikidump_filename, min_count, output_text_filename) self.emb_size = len(self.data.word2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.iteration = iteration self.initial_lr = initial_lr self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension) self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = Variable(torch.LongTensor(pos_u)) pos_v = Variable(torch.LongTensor(pos_v)) neg_v = Variable(torch.LongTensor(neg_v)) self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data, self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr def calculate_probability(self, word1, word2): embeddings = self.skip_gram_model.u_embeddings.weight.data.numpy() embedding_1 = embeddings[self.data.word2id[word1]] embedding_2 = embeddings[self.data.word2id[word2]] numerator = np.exp(np.sum(embedding_1 * embedding_2)) denominator = np.sum(np.exp( np.sum(np.multiply(embedding_1, embeddings), axis=1)), axis=0) return (numerator / denominator) def wordsim353_spearman(self, input_filename): target_word = [] context_word = [] human_scores = [] with open(input_filename) as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') ws353_pairs = -1 for row in csv_reader: if ws353_pairs == -1: ws353_pairs += 1 else: target_word.append(row[0]) context_word.append(row[1]) human_scores.append(float(row[2])) ws353_pairs += 1 for pair in range(0, ws353_pairs): if (target_word[pair] not in self.data.word2id): raise Exception('Target word not in model vocab: ', target_word[pair]) if (context_word[pair] not in self.data.word2id): raise Exception('Context word not in model vocab: ', context_word[pair]) human_rankings = ss.rankdata(human_scores) machine_scores = [] for pair in range(0, len(human_scores)): machine_scores.append( self.calculate_probability(target_word[pair], context_word[pair])) machine_rankings = ss.rankdata(machine_scores) human_scores_dict = dict() machine_scores_dict = dict() for pair in range(0, len(human_scores)): human_scores_dict[pair] = human_rankings[pair] machine_scores_dict[pair] = machine_rankings[pair] return spearman.spearman_correlation(human_scores_dict, machine_scores_dict)