def make_model(src_vocab,
tgt_vocab,
N=6,
d_model=512,
d_ff=2048,
h=8,
dropout=0.1):
attn = MultiHeadedAttention(h, d_model, dropout)
ff = PositionwiseFeedForward(d_model, d_ff, dropout)
pe = PositionalEncoding(d_model, dropout)
encoder_layer = EncoderLayer(d_model, copy(attn), copy(ff), dropout)
encoder = Encoder(encoder_layer, N)
decoder_layer = DecoderLayer(d_model, copy(attn), copy(attn), copy(ff), dropout)
decoder = Decoder(decoder_layer, N)
src_embed = nn.Sequential(Embedding(src_vocab, d_model), copy(pe))
tgt_embed = nn.Sequential(Embedding(tgt_vocab, d_model), copy(pe))
generator = Generator(d_model, tgt_vocab)
model = EncoderDecoder(encoder, decoder, src_embed, tgt_embed, generator)
for p in model.parameters():
if p.dim() > 1:
nn.init.xavier_uniform(p)
return model
def train(self, learning_rate=0.01, print_step=1000, stop_threshold=0):
losses = []
aver_losses = []
wa_scores = []
if print_step == 0:
print_step = self.n_batches
for _ in range(self.epochs):
iteration = 0
start = time.time()
with open(self.filename, 'r') as f:
reader = csv.reader(f)
for row in reader:
# Print step
iteration += 1
if iteration % print_step == 0:
end = time.time()
print("Epochs: {}".format(_),
"Iteration: {}".format(iteration),
"Avg. Training loss: {:.4f}".format(np.mean(losses)),
"{:.4f} sec/ {} sample".format((end - start), self.batch_size * print_step))
aver_losses.append(np.mean(losses))
losses = []
start = time.time()
# Print word analogy
if iteration % (print_step * 10) == 0:
eval = Embedding(np.array(self.E), self.int_to_vocab, self.vocab_to_int)
wa_score = self.word_analogy.evaluate(eval, high_level_category=False, restrict_top_words=False)
wa_scores.append(wa_score['all'])
self.export_model(self.output_dictionary + 'step-{}/'.format(int(iteration)))
loss = self._train_one_sample(int(row[0]), int(row[1]), learning_rate)
losses.append(loss)
eval = Embedding(np.array(self.E), self.int_to_vocab, self.vocab_to_int)
wa_score = self.word_analogy.evaluate(eval, high_level_category=False, restrict_top_words=False)
wa_scores.append(wa_score['all'])
print('Epochs: {}, WA score: {}'.format(_, wa_score['all']))
# Save step
if _ % 5 == 0:
self.export_model(self.output_dictionary + 'step-{}/'.format(int(_)))
# export losses
utils.save_pkl(aver_losses, self.output_dictionary + config['TRAIN']['loss_file'])
utils.save_pkl(wa_scores, self.output_dictionary + config['TRAIN']['acc_file'])
def load_attr_embeddings(self):
for attr_name in self.attr_embedding_map.keys():
print("load attr embedding for : {}".format(attr_name))
attr_embedding_info = self.attr_embedding_map[attr_name]
path = attr_embedding_info["path"]
cache_mod = attr_embedding_info["cache_mod"]
read_func = attr_embedding_info["read_func"]
attr_embedding = Embedding(path, cache_mod, read_func)
attr_embedding.load()
self.loaded_attr_embedding_map[attr_name] = attr_embedding
import torch.autograd as autograd
import torch.nn.functional as F
import torch.optim as optim
import time
from utils.dataset import Dataset
from utils.evaluation import Evaluation
from utils.embedding import Embedding
from utils.utils import stemming, vocabulary
from utils.utils import removeStopwords
import matplotlib
matplotlib.use('TkAgg')
import matplotlib.pyplot as plt
torch.manual_seed(1)
embedding = Embedding()
data = Dataset()
vocab = vocabulary(data.trainset)
word_to_ix = {word: i for i, word in enumerate(vocab)}
class RNN(nn.Module):
def __init__(self, embedding_dim, hidden_size, vocab_size):
super(RNN, self).__init__()
self.embedding_dim = embedding_dim
self.hidden_size = hidden_size
self.vocab_size = vocab_size
self.embedding = nn.Embedding(vocab_size + 1, embedding_dim)
self.embedding.weight.data.copy_(self.loadEmbedding())
self.t_lstm = nn.LSTM(embedding_dim,
hidden_size,
def train(self, print_step=1000, stop_threshold=0):
iteration = 1
loss = 0
losses = []
epoch_sum_loss = 0.
last_epoch_loss = 999999.
wa_scores = []
if print_step == 0:
print_step = self.n_batches
try:
start = time.time()
while True:
train_loss, _ = self.sess.run(
[self.full_cost, self.full_optimizer])
# train_loss, _ = self.sess.run([self.cost, self.optimizer])
loss += train_loss
epoch_sum_loss += train_loss
losses.append(train_loss)
if iteration % print_step == 0:
end = time.time()
print(
"Iteration: {}".format(iteration),
"Avg. Training loss: {:.4f}".format(loss / print_step),
"{:.4f} sec/ {} sample".format(
(end - start), self.batch_size * print_step))
loss = 0
start = time.time()
if iteration % self.n_batches == 0:
epochs = iteration / self.n_batches
epoch_loss = epoch_sum_loss / self.n_batches
epoch_sum_loss = 0
epoch_loss_diff = np.abs(epoch_loss - last_epoch_loss)
print('Epochs {} loss: {}'.format(epochs, epoch_loss))
# word analogy score
embedding = self.sess.run(self.embedding_g)
eval = Embedding(embedding, self.int_to_vocab,
self.vocab_to_int)
wa_score = self.word_analogy.evaluate(
eval,
high_level_category=False,
restrict_top_words=False)
wa_scores.append(wa_score['all'])
# stop criteria
if epoch_loss_diff < stop_threshold:
self.epochs = iteration / self.n_batches
# output file
self.embedding_file = config['TRAIN'][
'embedding'].format(self.n_embedding,
self.n_sampled,
int(self.epochs),
self.batch_size)
print('Loss diff: {}, stop training.'.format(
epoch_loss_diff))
print(self.output_dictionary + self.embedding_file)
break
# Save step
if epochs % 10 == 0:
self.embedding = self.sess.run(self.embedding_g)
self.softmax_w = self.sess.run(self.softmax_w_g)
self.export_model(self.output_dictionary +
'step-{}/'.format(int(epochs)))
last_epoch_loss = epoch_loss
iteration += 1
except tf.errors.OutOfRangeError:
print("End of dataset")
# export embedding matrix
self.embedding = self.sess.run(self.embedding_g)
self.softmax_w = self.sess.run(self.softmax_w_g)
# export losses
utils.save_pkl(losses,
self.output_dictionary + config['TRAIN']['loss_file'])
utils.save_pkl(wa_scores,
self.output_dictionary + config['TRAIN']['acc_file'])