def train(self):
model = skipgram(self.vocabulary_size, self.embedding_dim)
if torch.cuda.is_available():
model.cuda()
optimizer = optim.SGD(model.parameters(), lr=0.2)
for epoch in range(self.epoch_num):
start = time.time()
self.op.process = True
batch_num = 0
batch_new = 0
while self.op.process:
pos_u, pos_v, neg_v = self.op.generate_batch(self.windows_size, self.batch_size, self.neg_sample_num)
pos_u = Variable(torch.LongTensor(pos_u))
pos_v = Variable(torch.LongTensor(pos_v))
neg_v = Variable(torch.LongTensor(neg_v))
if torch.cuda.is_available():
pos_u = pos_u.cuda()
pos_v = pos_v.cuda()
neg_v = neg_v.cuda()
optimizer.zero_grad()
loss = model(pos_u, pos_v, neg_v, self.batch_size)
loss.backward()
optimizer.step()
if batch_num % 30000 == 0:
torch.save(model.state_dict(), './tmp/skipgram.epoch{}.batch{}'.format(epoch, batch_num))
if batch_num % 1000 == 0:
end = time.time()
# word_embeddings = model.input_embeddings()
## sp1 and sp2 based in distinct words
# sp1, sp2 = scorefunction(word_embeddings)
## loss,data[0] to loss.data
# print('eporch,batch=%2d %5d: sp=%1.3f %1.3f pair/sec = %4.2f loss=%4.3f\r' \
# % (epoch, batch_num, sp1, sp2, (batch_num - batch_new) * self.batch_size / (end - start),
# loss.data), end="")
print('eporch,batch=%2d %5d: pair/sec = %4.2f loss=%4.3f\r' \
% (epoch, batch_num,
(batch_num - batch_new) * self.batch_size / (end - start),
loss.data), end="")
batch_new = batch_num
start = time.time()
print()
batch_num = batch_num + 1
# saving each epoch
# bell
print('\a')
model.save_embedding(os.path.join("data",
"embed_epoch_" + str(epoch) + ".vec"),
self.op.dic_idx2word)
print()
print("Optimization Finished!")
def train(self, embed_filename):
model = skipgram(self.vocabulary_size, self.embedding_dim)
if torch.cuda.is_available():
model.cuda()
optimizer = optim.SGD(model.parameters(), lr=0.2)
for epoch in range(self.epoch_num):
start = time.time()
self.op.process = True
batch_num = 0
batch_new = 0
while self.op.process:
pos_u, pos_v, neg_v = self.op.generate_batch(
self.windows_size, self.batch_size, self.neg_sample_num)
pos_u = Variable(torch.LongTensor(pos_u))
pos_v = Variable(torch.LongTensor(pos_v))
neg_v = Variable(torch.LongTensor(neg_v))
if torch.cuda.is_available():
pos_u = pos_u.cuda()
pos_v = pos_v.cuda()
neg_v = neg_v.cuda()
optimizer.zero_grad()
loss = model(pos_u, pos_v, neg_v, self.batch_size)
loss.backward()
optimizer.step()
if batch_num % 30000 == 0:
torch.save(
model.state_dict(),
'./tmp/skipgram.epoch{}.batch{}'.format(
epoch, batch_num))
if batch_num % 2000 == 0:
end = time.time()
word_embeddings = model.input_embeddings()
sp1, sp2 = scorefunction(word_embeddings)
print(
'eporch,batch=%2d %5d: sp=%1.3f %1.3f pair/sec = %4.2f loss=%4.3f\r'
% (epoch, batch_num, sp1, sp2,
(batch_num - batch_new) * self.batch_size /
(end - start), loss.data),
end="")
batch_new = batch_num
start = time.time()
batch_num = batch_num + 1
print()
model.save_embedding(embed_filename, self.op.vocab_words)
print("Optimization Finished!")
def train(self, pre_trained_model):
model = skipgram(self.vocabulary_size, self.embedding_dim, pre_trained_model)
if torch.cuda.is_available():
model.cuda()
optimizer = optim.SGD(model.parameters(),lr=0.2)
loss_history = list()
for epoch in range(self.epoch_num):
start = time.time()
self.op.process = True
batch_num = 0
batch_new = 0
while self.op.process:
pos_u, pos_v, neg_v = self.op.generate_batch(self.windows_size, self.batch_size, self.neg_sample_num)
pos_u = Variable(torch.LongTensor(pos_u))
pos_v = Variable(torch.LongTensor(pos_v))
neg_v = Variable(torch.LongTensor(neg_v))
if torch.cuda.is_available():
pos_u = pos_u.cuda()
pos_v = pos_v.cuda()
neg_v = neg_v.cuda()
optimizer.zero_grad()
loss = model(pos_u, pos_v, neg_v,self.batch_size)
loss.backward()
optimizer.step()
if batch_num % 10 == 0:
loss_history.append(loss.data[0])
if batch_num % 2000 == 0:
end = time.time()
word_embeddings = model.input_embeddings()
# sp1, sp2 = scorefunction(word_embeddings)
print('epoch,batch=%2d %5d: pair/sec = %4.2f loss=%4.3f\r', \
epoch, batch_num, (batch_num - batch_new) * self.batch_size / (end - start), loss.data[0])
batch_new = batch_num
start = time.time()
batch_num = batch_num + 1
print()
torch.save(model.state_dict(), __location__ + '/skipgram.epoch{}.batch{}'.format(epoch, batch_num))
plt.plot(loss_history[::100])
plt.ylabel('loss (stat.ML)')
plt.show()
print("Optimization Finished!")
def train(self):
model = skipgram(self.vocabulary_size, self.embedding_dim)
if torch.cuda.is_available():
print("CUDA available")
model.cuda()
else:
print("CUDA NOT available")
controler = 0
optimizer = optim.SGD(model.parameters(),lr=0.2)
#for epoch in range(self.epoch_num):
for epoch in range(self.epoch_num):
start = time.time()
self.op.process = True
batch_num = 0
batch_new = 0
while self.op.process:
batch_data, neg_v = self.op.generate_batch(self.batch_size, self.neg_sample_num)
pos_u = [pair[0] for pair in batch_data]
pos_v = [pair[1] for pair in batch_data]
length = []
for i in map(len,pos_u):
length.append(i)
length = torch.LongTensor(length)
data_tensor = Variable(torch.zeros((len(pos_u), length.max()))).long()
for idx, (seq, datalen) in enumerate(zip(pos_u, length)):
data_tensor[idx, :datalen] = torch.LongTensor(seq)
pos_u = data_tensor
pos_v = Variable(torch.LongTensor(pos_v))
neg_v = Variable(torch.LongTensor(neg_v))
length = length.expand(self.embedding_dim,self.batch_size).transpose(0,1)
length = Variable(length).float()
if torch.cuda.is_available():
pos_u = pos_u.cuda()
pos_v = pos_v.cuda()
neg_v = neg_v.cuda()
length = length.cuda()
optimizer.zero_grad()
loss = model.forward(pos_u, pos_v, neg_v, length, self.embedding_dim)
loss.backward()
optimizer.step()
if batch_num%2000 == 0:
end = time.time()
word_embeddings = model.input_embeddings()
sp1, sp2, num = scorefunction(word_embeddings)
print('eporch,batch=%2d %5d: sp=%1.3f %1.3f %3d pair/sec = %4.2f loss=%4.3f\r'%(epoch, batch_num, sp1, sp2, num, (batch_num-batch_new)*self.batch_size/(end-start),loss.data[0])
, end="")
batch_new = batch_num
start = time.time()
batch_num = batch_num + 1
print()
if epoch == 10: model.save_embedding('embed_10_b2048_top50t3_120w.txt', self.op.id2word)
print("Optimization Finished!")
model.save_embedding('embed_20_b2048_top50t3_120w.txt', self.op.id2word)
def train(self):
# cudnn.benchmark = True
model = skipgram(self.vocabulary_size, self.embedding_dim)
if torch.cuda.is_available():
print("using cuda")
model.cuda()
else:
print("not using cuda")
optimizer = optim.SGD(model.parameters(), lr=0.2)
for epoch in range(self.epoch_num):
start = time.time()
self.op.process = True
batch_num = 0
batch_new = 0
while self.op.process:
pos_u, pos_v, neg_v = self.op.generate_batch(
self.windows_size, self.batch_size, self.neg_sample_num)
pos_u = Variable(torch.LongTensor(pos_u))
pos_v = Variable(torch.LongTensor(pos_v))
neg_v = Variable(torch.LongTensor(neg_v))
if torch.cuda.is_available():
pos_u = pos_u.cuda()
pos_v = pos_v.cuda()
neg_v = neg_v.cuda()
optimizer.zero_grad()
loss = model(pos_u, pos_v, neg_v, self.batch_size)
loss.backward()
optimizer.step()
if batch_num % 30000 == 0:
torch.save(
model.state_dict(),
'./tmp/skipgram.epoch{}.batch{}'.format(
epoch, batch_num))
if batch_num % 2000 == 0:
end = time.time()
word_embeddings = model.input_embeddings()
# sp1, sp2 = scorefunction(word_embeddings)
print('epoch,batch=%2d %5d: pair/sec = %4.2f loss=%4.3f\r'\
%(epoch, batch_num, (batch_num-batch_new)*self.batch_size/(end-start),loss.data),end="")
batch_new = batch_num
start = time.time()
batch_num = batch_num + 1
print()
tsne = TSNE(perplexity=30, n_components=2, init='random', n_iter=5000)
embeds = model.u_embeddings.weight.data
labels = []
tokens = []
max_size = 1000
for idx in range(min(len(embeds), len(self.op.vocab_words), max_size)):
tokens.append(embeds[idx].cpu().numpy())
labels.append(self.op.vocab_words[idx])
pca = PCA(n_components=50)
pca_result = pca.fit_transform(tokens)
low_dim_embs = tsne.fit_transform(pca_result)
self.plot_with_labels(low_dim_embs, labels, 'tsne.png')
print("Optimization finished!")
def train(self):
model = skipgram(self.vocabulary_size, self.embedding_dim)
if torch.cuda.is_available():
model.cuda()
#return model
optimizer = optim.SGD(model.parameters(), lr=0.2)
for epoch in range(self.epoch_num):
epoch_start = time.time()
start = time.time()
self.op.process = True
batch_num = 0
batch_new = 0
while self.op.process:
pos_u, pos_v, neg_v = self.op.generate_batch(
self.windows_size,
self.batch_size,
self.neg_sample_num,
verbose=False)
pos_u = Variable(torch.LongTensor(pos_u))
pos_v = Variable(torch.LongTensor(pos_v))
neg_v = Variable(torch.LongTensor(neg_v))
if torch.cuda.is_available():
pos_u = pos_u.cuda()
pos_v = pos_v.cuda()
neg_v = neg_v.cuda()
optimizer.zero_grad()
loss = model(pos_u, pos_v, neg_v, self.batch_size)
loss.backward()
optimizer.step()
if batch_num % 2000 == 0:
end = time.time()
with torch.no_grad():
total_val_loss = 0.
n_val_batch = 0
self.val_op.process = True
while self.val_op.process:
pos_u, pos_v, neg_v = self.val_op.generate_batch(
self.windows_size, self.batch_size,
self.neg_sample_num)
pos_u = Variable(torch.LongTensor(pos_u))
pos_v = Variable(torch.LongTensor(pos_v))
neg_v = Variable(torch.LongTensor(neg_v))
if torch.cuda.is_available():
pos_u = pos_u.cuda()
pos_v = pos_v.cuda()
neg_v = neg_v.cuda()
val_loss = model(pos_u, pos_v, neg_v,
self.batch_size)
total_val_loss += val_loss.item()
n_val_batch += 1
word_embeddings = model.input_embeddings()
print('epoch,batch=%2d %5d: pair/sec = %4.2f loss=%4.3f val_loss=%4.3f\r'\
%(epoch, batch_num, (batch_num-batch_new)*self.batch_size/(end-start),loss.item(), total_val_loss/n_val_batch))
batch_new = batch_num
start = time.time()
batch_num = batch_num + 1
print("epoch stat, time: %.2f, batch_num: %d" %
(time.time() - epoch_start, batch_num))
torch.save(model.state_dict(),
'./tmp/skipgram.epoch{}'.format(epoch))
print("Optimization Finished!")
