forked from Andras7/word2vec-pytorch
/
trainer.py
67 lines (51 loc) · 2.52 KB
/
trainer.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
import torch
import torch.optim as optim
from torch.utils.data import DataLoader
from tqdm import tqdm
from data_reader import DataReader, Word2vecDataset
from model import SkipGramModel
class Word2VecTrainer:
def __init__(self, input_file, output_file, emb_dimension=300, batch_size=64, window_size=5, iterations=5,
initial_lr=1.0, min_count=5):
self.data = DataReader(input_file, min_count)
dataset = Word2vecDataset(self.data, window_size)
self.dataloader = DataLoader(dataset, batch_size=batch_size,
shuffle=False, num_workers=0, collate_fn=dataset.collate)
self.output_file_name = output_file
self.emb_size = len(self.data.word2id)
self.emb_dimension = emb_dimension
self.batch_size = batch_size
self.iterations = iterations
self.initial_lr = initial_lr
self.skip_gram_model = SkipGramModel(self.emb_size, self.emb_dimension)
self.use_cuda = torch.cuda.is_available()
self.device = torch.device("cuda" if self.use_cuda else "cpu")
if self.use_cuda:
print("USING CUDA")
self.skip_gram_model.cuda()
else:
print("CUDA FAIL")
def train(self):
for iteration in range(self.iterations):
print("\n\n\nIteration: " + str(iteration + 1))
optimizer = optim.SGD(self.skip_gram_model.parameters(), lr=self.initial_lr)
# scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, len(self.dataloader))
running_loss = 0.0
for i, sample_batched in enumerate(tqdm(self.dataloader)):
if len(sample_batched[0]) > 1:
pos_u = sample_batched[0].to(self.device)
pos_v = sample_batched[1].to(self.device)
neg_v = sample_batched[2].to(self.device)
# scheduler.step()
optimizer.zero_grad()
loss = self.skip_gram_model.forward(pos_u, pos_v, neg_v)
loss.backward()
optimizer.step()
running_loss = running_loss * 0.95 + loss.item() * 0.05
if i > 0 and i % 400 == 0:
print(" Loss: " + str(running_loss))
self.skip_gram_model.save_embedding(self.data.id2word, self.output_file_name.format(iteration))
self.initial_lr *= 0.7
if __name__ == '__main__':
w2v = Word2VecTrainer(input_file="corpus.txt", output_file="pw2v-{}.bin")
w2v.train()