class Word2Vec: def __init__(self, input_file_name, output_file_name): self.min_count = 5 self.emb_dimension = 100 self.batch_size = 64 self.window_size = 5 self.iteration = 1 self.initial_lr = 0.001 self.data = InputData(input_file_name, self.min_count) self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension, self.batch_size, self.window_size, self.iteration, self.initial_lr, self.min_count) self.use_cuda = torch.cuda.is_available() if self.use_cuda: self.skip_gram_model.cuda() self.optimizer = optim.SGD( self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): """Multiple training. Returns: None. """ pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = Variable(torch.LongTensor(pos_u)) pos_v = Variable(torch.LongTensor(pos_v)) neg_v = Variable(torch.LongTensor(neg_v)) if self.use_cuda: pos_u = pos_u.cuda() pos_v = pos_v.cuda() neg_v = neg_v.cuda() self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() process_bar.set_description("Loss: %0.8f, lr: %0.6f" % (loss.data, self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr self.skip_gram_model.save_embedding( self.data.id2word, self.output_file_name, self.use_cuda)
class Word2Vec: def __init__(self, input_file_name, output_file_name, emb_dimension=100, batch_size=50, window_size=5, iteration=5, initial_lr=0.025, neg_num=5, min_count=5): self.data = InputData(input_file_name, min_count) self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.iteration = iteration self.initial_lr = initial_lr self.neg_num = neg_num self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension) self.skip_gram_model.cuda() self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) count = int(batch_count) // 3 for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, self.neg_num) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = Variable(torch.LongTensor(pos_u)).cuda() pos_v = Variable(torch.LongTensor(pos_v)).cuda() neg_v = Variable(torch.LongTensor(neg_v)).cuda() self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() process_bar.set_description("Loss: %0.8f, lr: %0.6f" % (loss.item(), self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr if i != 0 and i % count == 0: self.skip_gram_model.save_embedding(self.data.id2word,self.output_file_name + str(i)) self.skip_gram_model.save_embedding(self.data.id2word, self.output_file_name + 'final')
class Word2Vec: def __init__(self, input_file_name, output_file_name, emb_dimension=100, batch_size=50, window_size=5, iteration=1, initial_lr=0.025, min_count=1): """Initilize class parameters. Args: input_file_name: Name of a text data from file. Each line is a sentence splited with space. output_file_name: Name of the final embedding file. emb_dimention: Embedding dimention, typically from 50 to 500. batch_size: The count of word pairs for one forward. window_size: Max skip length between words. iteration: Control the multiple training iterations. initial_lr: Initial learning rate. min_count: The minimal word frequency, words with lower frequency will be filtered. Returns: None. """ self.data = InputData(input_file_name, min_count) self.output_file_name = output_file_name self.emb_size = len(self.data.word2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.iteration = iteration self.initial_lr = initial_lr self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension) self.use_cuda = torch.cuda.is_available() if self.use_cuda: self.skip_gram_model.cuda() self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): """Multiple training. Returns: None. """ pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) # self.skip_gram_model.save_embedding( # self.data.id2word, 'begin_embedding.txt', self.use_cuda) for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = Variable(torch.LongTensor(pos_u)) pos_v = Variable(torch.LongTensor(pos_v)) neg_v = Variable(torch.LongTensor(neg_v)) if self.use_cuda: pos_u = pos_u.cuda() pos_v = pos_v.cuda() neg_v = neg_v.cuda() self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr self.skip_gram_model.save_embedding(self.data.id2word, self.output_file_name, self.use_cuda)
class Word2Vec: def __init__( self, input_path, output_dir, wordsim_path, dimension=100, batch_size=batch_size, window_size=5, epoch_count=1, initial_lr=1e-6, min_count=5, ): self.data = InputData(input_path, min_count) self.output_dir = output_dir self.vocabulary_size = len(self.data.id_from_word) self.dimension = dimension self.batch_size = batch_size self.window_size = window_size self.epoch_count = epoch_count self.initial_lr = initial_lr self.model = SkipGramModel(self.vocabulary_size, self.dimension) if torch.cuda.is_available(): self.device = torch.device('cuda') else: self.device = torch.device('cpu') self.model = nn.DataParallel(self.model.to(self.device)) self.optimizer = optim.SGD(self.model.parameters(), lr=self.initial_lr) if wordsim_path: self.wordsim_verification_tuples = [] with open(wordsim_path, 'r') as f: f.readline() # Abandon header for line in f: word1, word2, actual_similarity = line.split(',') self.wordsim_verification_tuples.append( (word1, word2, float(actual_similarity)) ) else: self.wordsim_verification_tuples = None def train(self): pair_count = self.data.get_pair_count(self.window_size) batch_count = self.epoch_count * pair_count / self.batch_size best_rho = float('-inf') for i in tqdm(range(int(batch_count)), total=batch_count): self.model.train() pos_pairs = self.data.get_batch_pairs( self.batch_size, self.window_size ) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = torch.tensor(pos_u, device=self.device) pos_v = torch.tensor(pos_v, device=self.device) neg_v = torch.tensor(neg_v, device=self.device) self.optimizer.zero_grad() loss = self.model(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() if i % 250 == 0: self.model.eval() rho = self.model.module.get_wordsim_rho( self.wordsim_verification_tuples, self.data.id_from_word, self.data.word_from_id ) print( f'Loss: {loss.item()},' f' lr: {self.optimizer.param_groups[0]["lr"]},' f' rho: {rho}' ) dump_embedding( self.model.module.get_embedding( self.data.id_from_word, self.data.word_from_id ), self.model.module.dimension, self.data.word_from_id, os.path.join(self.output_dir, f'latest.txt'), ) if rho > best_rho: dump_embedding( self.model.module.get_embedding( self.data.id_from_word, self.data.word_from_id ), self.model.module.dimension, self.data.word_from_id, os.path.join(self.output_dir, f'{i}_{rho}.txt') ) best_rho = rho # warm up if i < 10000: lr = self.initial_lr * i / 10000 for param_group in self.optimizer.param_groups: param_group['lr'] = lr elif i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr
class Word2Vec: def __init__(self, output_file_name, output_sense_name, emb_dimension=128, K=5, batch_size=1, window_size=5, iteration=1, initial_lr=0.1, createClusterLambda=1.5, min_count=0): """Initilize class parameters. Args: input_file_name: Name of a text data from file. Each line is a sentence splited with space. output_file_name: Name of the final embedding file. emb_dimention: Embedding dimention, typically from 50 to 500. batch_size: The count of word pairs for one forward. window_size: Max skip length between words. iteration: Control the multiple training iterations. initial_lr: Initial learning rate. min_count: The minimal word frequency, words with lower frequency will be filtered. Returns: None. """ self.data = InputData(min_count) self.output_file_name = output_file_name self.output_sense_name = output_sense_name self.emb_size = len(self.data.node2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.K = K self.iteration = iteration self.initial_lr = initial_lr self.createClusterLambda = createClusterLambda self.skip_gram_model = SkipGramModel(self.emb_size, self.K, self.emb_dimension) self.use_cuda = torch.cuda.is_available() if self.use_cuda: self.skip_gram_model.cuda() self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): """Multiple training. Returns: None. """ pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) total_pos_pairs = self.data.get_node_pairs(self.window_size) print("training\n") for t in process_bar: pos_pairs = total_pos_pairs[t] neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] # right=[] cnt = 0 curword = pos_u[cnt] contextwords = [] contextwords_cuda = [] while cnt < len(pos_u): contextwords.append(pos_v[cnt]) contextwords_cuda.append(pos_v[cnt]) cnt += 1 contextembedding = torch.zeros(self.emb_dimension) contextwords_cuda = Variable(torch.LongTensor(contextwords_cuda)) if self.use_cuda: contextwords_cuda = contextwords_cuda.cuda() emb_v = self.skip_gram_model.v_embeddings(contextwords_cuda) if self.use_cuda: emb_v_data = emb_v.cpu().data else: emb_v_data = emb_v.data for i in range(len(contextwords)): contextembedding += emb_v_data[i] # torch.add(contextembedding,emb_v_data[i,:],out=emb_v_data_total) emb_v_data_avg = contextembedding / (len(contextwords)) # torch.div(emb_v_data_total,len(contextwords),out=emb_v_data_avg) minDist = np.inf rightsense = 0 mu = torch.Tensor(self.emb_dimension) if self.skip_gram_model.num_sense[curword] == self.K: nC = self.K else: nC = self.skip_gram_model.num_sense[curword] + 1 prob = torch.Tensor(nC) for k in range(self.skip_gram_model.num_sense[curword]): torch.div(self.skip_gram_model.clusterCenter[curword, k, :], self.skip_gram_model.clusterCount[curword][k], out=mu) x_norm = torch.norm(emb_v_data_avg, p=2) y_norm = torch.norm(mu, p=2) summ = 0 for p in range(self.emb_dimension): summ += emb_v_data_avg[p] * mu[p] dist = 1 - summ / (x_norm * y_norm) prob[k] = dist if dist < minDist: minDist = dist rightsense = k if self.skip_gram_model.num_sense[curword] < self.K: if self.createClusterLambda < minDist: prob[self.skip_gram_model. num_sense[curword]] = self.createClusterLambda rightsense = self.skip_gram_model.num_sense[curword] self.skip_gram_model.num_sense[curword] += 1 for i in range(self.emb_dimension): self.skip_gram_model.clusterCenter[curword][rightsense][ i] += emb_v_data_avg[i] self.skip_gram_model.clusterCount[curword][rightsense] += 1 # for i in range(len(contextwords)): # right.append(rightsense) self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v, rightsense, self.use_cuda) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data[0], self.optimizer.param_groups[0]['lr'])) if t * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * t / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr self.skip_gram_model.save_embedding(self.data.id2node, self.output_file_name, self.output_sense_name, self.use_cuda)
class Word2Vec: def __init__(self, wikidump_filename, output_text_filename, emb_dimension, batch_size, window_size, iteration, initial_lr, min_count): self.data = InputData(wikidump_filename, min_count, output_text_filename) self.emb_size = len(self.data.word2id) self.emb_dimension = emb_dimension self.batch_size = batch_size self.window_size = window_size self.iteration = iteration self.initial_lr = initial_lr self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension) self.optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr) def train(self): pair_count = self.data.evaluate_pair_count(self.window_size) batch_count = self.iteration * pair_count / self.batch_size process_bar = tqdm(range(int(batch_count))) for i in process_bar: pos_pairs = self.data.get_batch_pairs(self.batch_size, self.window_size) neg_v = self.data.get_neg_v_neg_sampling(pos_pairs, 5) pos_u = [pair[0] for pair in pos_pairs] pos_v = [pair[1] for pair in pos_pairs] pos_u = Variable(torch.LongTensor(pos_u)) pos_v = Variable(torch.LongTensor(pos_v)) neg_v = Variable(torch.LongTensor(neg_v)) self.optimizer.zero_grad() loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v) loss.backward() self.optimizer.step() process_bar.set_description( "Loss: %0.8f, lr: %0.6f" % (loss.data, self.optimizer.param_groups[0]['lr'])) if i * self.batch_size % 100000 == 0: lr = self.initial_lr * (1.0 - 1.0 * i / batch_count) for param_group in self.optimizer.param_groups: param_group['lr'] = lr def calculate_probability(self, word1, word2): embeddings = self.skip_gram_model.u_embeddings.weight.data.numpy() embedding_1 = embeddings[self.data.word2id[word1]] embedding_2 = embeddings[self.data.word2id[word2]] numerator = np.exp(np.sum(embedding_1 * embedding_2)) denominator = np.sum(np.exp( np.sum(np.multiply(embedding_1, embeddings), axis=1)), axis=0) return (numerator / denominator) def wordsim353_spearman(self, input_filename): target_word = [] context_word = [] human_scores = [] with open(input_filename) as csv_file: csv_reader = csv.reader(csv_file, delimiter=',') ws353_pairs = -1 for row in csv_reader: if ws353_pairs == -1: ws353_pairs += 1 else: target_word.append(row[0]) context_word.append(row[1]) human_scores.append(float(row[2])) ws353_pairs += 1 for pair in range(0, ws353_pairs): if (target_word[pair] not in self.data.word2id): raise Exception('Target word not in model vocab: ', target_word[pair]) if (context_word[pair] not in self.data.word2id): raise Exception('Context word not in model vocab: ', context_word[pair]) human_rankings = ss.rankdata(human_scores) machine_scores = [] for pair in range(0, len(human_scores)): machine_scores.append( self.calculate_probability(target_word[pair], context_word[pair])) machine_rankings = ss.rankdata(machine_scores) human_scores_dict = dict() machine_scores_dict = dict() for pair in range(0, len(human_scores)): human_scores_dict[pair] = human_rankings[pair] machine_scores_dict[pair] = machine_rankings[pair] return spearman.spearman_correlation(human_scores_dict, machine_scores_dict)