def runTest(args, n_layers, hidden_size, reverse, modelFile, beam_size,
batch_size, input, corpus):
data, length = loadPrepareData(args)
voc = data.voc
print('load data...')
user_length, item_length = length #, user_length2, item_length2 = length
# train_batches = batchify(data.train, data.user_text, user_length, data.item_text, item_length, batch_size)
# val_batches = batchify(data.dev, data.user_text, user_length, data.item_text, item_length, batch_size)
test_batches = batchify(data.test, data.user_text, user_length,
data.item_text, item_length, batch_size)
print('Building encoder and decoder ...')
embedding = nn.Embedding(data.voc.n_words, hidden_size)
encoderU = EncoderRNNlinear(data.voc.n_words, hidden_size, embedding,
data.dmax, n_layers)
encoderB = EncoderRNNlinear(data.voc.n_words, hidden_size, embedding,
data.dmax, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
data.voc.n_words, n_layers)
# load model
checkpoint = torch.load(modelFile)
encoderU.load_state_dict(checkpoint['enU'])
encoderB.load_state_dict(checkpoint['enB'])
decoder.load_state_dict(checkpoint['de'])
# train mode set to false, effect only on dropout, batchNorm
encoderU.train(False)
encoderB.train(False)
decoder.train(False)
if USE_CUDA:
encoderU = encoderU.cuda()
encoderB = encoderB.cuda()
decoder = decoder.cuda()
if not args.sample:
# evaluate on test
# for test_batch in tqdm(test_batches):
for test_i, test_batch in enumerate(test_batches):
if test_i > 1:
break
input_index, input_variable, lengths, target_variable, mask, max_target_len = test_batch
user_input_variable, business_input_variable = input_variable
user_lengths, business_lengths = lengths
# evaluate on train
evaluateRandomly(encoderU, encoderB, decoder, voc, \
input_index, user_input_variable, business_input_variable, \
user_lengths, business_lengths, \
target_variable, mask, max_target_len, reverse, beam_size)
else:
# evaluate using sample
sample(encoderU, encoderB, decoder, voc, test_batches, reverse)
def trainIters(corpus,
reverse,
n_iteration,
learning_rate,
batch_size,
n_layers,
hidden_size,
print_every,
save_every,
loadFilename=None,
attn_model='dot',
decoder_learning_ratio=5.0):
voc, pairs = loadPrepareData(corpus)
# training data
corpus_name = os.path.split(corpus)[-1].split('.')[0]
training_batches = None
try:
training_batches = torch.load(os.path.join(save_dir, 'training_data', corpus_name,
'{}_{}_{}.tar'.format(n_iteration, \
filename(reverse, 'training_batches'), \
batch_size)))
except BaseException: #OWEN: was FileNotFoundError
print('Training pairs not found, generating ...')
training_batches = [
batch2TrainData(voc,
[random.choice(pairs)
for _ in range(batch_size)], reverse)
for _ in range(n_iteration)
]
torch.save(training_batches, os.path.join(save_dir, 'training_data', corpus_name,
'{}_{}_{}.tar'.format(n_iteration, \
filename(reverse, 'training_batches'), \
batch_size)))
# model
checkpoint = None
print('Building encoder and decoder ...')
embedding = nn.Embedding(voc.n_words, hidden_size)
encoder = EncoderRNN(voc.n_words, hidden_size, embedding, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
voc.n_words, n_layers)
if loadFilename:
checkpoint = torch.load(loadFilename)
encoder.load_state_dict(checkpoint['en'])
decoder.load_state_dict(checkpoint['de'])
# use cuda
if USE_CUDA:
encoder = encoder.cuda()
decoder = decoder.cuda()
# optimizer
print('Building optimizers ...')
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
if loadFilename:
encoder_optimizer.load_state_dict(checkpoint['en_opt'])
decoder_optimizer.load_state_dict(checkpoint['de_opt'])
# initialize
print('Initializing ...')
start_iteration = 1
perplexity = []
print_loss = 0
if loadFilename:
start_iteration = checkpoint['iteration'] + 1
perplexity = checkpoint['plt']
for iteration in tqdm(range(start_iteration, n_iteration + 1)):
training_batch = training_batches[iteration - 1]
input_variable, lengths, target_variable, mask, max_target_len = training_batch
loss = train(input_variable, lengths, target_variable, mask,
max_target_len, encoder, decoder, embedding,
encoder_optimizer, decoder_optimizer, batch_size)
print_loss += loss
perplexity.append(loss)
if iteration % print_every == 0:
print_loss_avg = math.exp(print_loss / print_every)
perplexity.append(print_loss_avg)
# show perplexity (lots of numbers!):
#print(perplexity, iteration)
# plotPerplexity(perplexity, iteration)
print('%d %d%% %.4f' %
(iteration, iteration / n_iteration * 100, print_loss_avg))
print_loss = 0
if (iteration % save_every == 0):
directory = os.path.join(
save_dir, 'model', corpus_name,
'{}-{}_{}'.format(n_layers, n_layers, hidden_size))
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'iteration': iteration,
'en': encoder.state_dict(),
'de': decoder.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'plt': perplexity
},
os.path.join(
directory,
'{}_{}.tar'.format(iteration,
filename(reverse,
'backup_bidir_model'))))
def trainIters(corpus,
pre_modelFile,
reverse,
n_iteration,
learning_rate,
batch_size,
n_layers,
hidden_size,
print_every,
save_every,
loadFilename=None,
attn_model='dot',
decoder_learning_ratio=5.0):
voc, pairs = loadPrepareData(corpus)
# training data
corpus_name = os.path.split(corpus)[-1].split('.')[0]
training_batches = None
try:
training_batches = torch.load(os.path.join(save_dir, 'training_data', corpus_name,
'{}_{}_{}.tar'.format(n_iteration, \
filename(reverse, 'training_batches'), \
batch_size)))
except FileNotFoundError:
print('Training pairs not found, generating ...')
training_batches = [
batch2TrainData(voc,
[random.choice(pairs)
for _ in range(batch_size)], reverse)
for _ in range(n_iteration)
]
torch.save(training_batches, os.path.join(save_dir, 'training_data', corpus_name,
'{}_{}_{}.tar'.format(n_iteration, \
filename(reverse, 'training_batches'), \
batch_size)))
# model
checkpoint = None
#print('Building pretrained word2vector model...')
embedding = nn.Embedding(
300, hidden_size) #The dimension of google's model is 300
#-----------------------------------------------------------------
#my code
'''
EMBEDDING_DIM = 300 #Should be the same as hidden_size!
if EMBEDDING_DIM != hidden_size:
sys.exit("EMBEDDING_DIM do not equal to hidden_size. Please correct it.")
CONTEXT_SIZE = 2
pre_checkpoint = torch.load(pre_modelFile)
pretrained_model = NGramLanguageModeler(voc.n_words, EMBEDDING_DIM, CONTEXT_SIZE)
pretrained_model.load_state_dict(pre_checkpoint['w2v'])
pretrained_model.train(False)
embedding = pretrained_model
'''
if USE_CUDA:
embedding = embedding.cuda()
#-----------------------------------------------------------------
#replace embedding by pretrained_model
print('Building encoder and decoder ...')
encoder = EncoderRNN(300, hidden_size, embedding, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
voc.n_words, n_layers)
if loadFilename:
checkpoint = torch.load(loadFilename)
encoder.load_state_dict(checkpoint['en'])
decoder.load_state_dict(checkpoint['de'])
# use cuda
if USE_CUDA:
encoder = encoder.cuda()
decoder = decoder.cuda()
# optimizer
print('Building optimizers ...')
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
if loadFilename:
encoder_optimizer.load_state_dict(checkpoint['en_opt'])
decoder_optimizer.load_state_dict(checkpoint['de_opt'])
# Load Google's pre-trained Word2Vec model.
print('Loading w2v_model ...')
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s',
level=logging.INFO)
w2v_model = gensim.models.KeyedVectors.load_word2vec_format(pre_modelFile,
binary=True)
print("Loading complete!")
# initialize
print('Initializing ...')
start_iteration = 1
perplexity = []
print_loss = 0
if loadFilename:
start_iteration = checkpoint['iteration'] + 1
perplexity = checkpoint['plt']
for iteration in tqdm(range(start_iteration, n_iteration + 1)):
training_batch = training_batches[iteration - 1]
input_variable, lengths, target_variable, mask, max_target_len = training_batch
loss = train(input_variable, lengths, target_variable, mask,
max_target_len, encoder, decoder, embedding,
encoder_optimizer, decoder_optimizer, batch_size,
w2v_model, voc)
print_loss += loss
perplexity.append(loss)
if iteration % print_every == 0:
print_loss_avg = math.exp(print_loss / print_every)
# perplexity.append(print_loss_avg)
# plotPerplexity(perplexity, iteration)
print('%d %d%% %.4f' %
(iteration, iteration / n_iteration * 100, print_loss_avg))
print_loss = 0
if (iteration % save_every == 0):
directory = os.path.join(
save_dir, 'model', corpus_name,
'{}-{}_{}'.format(n_layers, n_layers, hidden_size))
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'iteration': iteration,
'en': encoder.state_dict(),
'de': decoder.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'plt': perplexity
},
os.path.join(
directory,
'{}_{}.tar'.format(iteration,
filename(reverse,
'backup_bidir_model'))))
decoder = LuongAttnDecoderRNN(attn_model,
hidden_size,
len(dictionary),
n_layers,
dropout=dropout)
# Initialize optimizers and criterion
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
criterion = nn.CrossEntropyLoss()
# Move models to GPU
if USE_CUDA:
encoder.cuda()
decoder.cuda()
# =============
# Begin!
# =============
print_loss_total = 0.0
start = time.time()
while epoch < n_epochs:
epoch += 1
# Get training data for this cycle
input_batches, input_lengths, target_batches, target_lengths = random_batch(
raw_data, batch_size, dictionary)
# Run the train function
def main():
epoch = 1000
batch_size = 64
hidden_dim = 300
use_cuda = True
encoder = Encoder(num_words, hidden_dim)
if args.attn:
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, hidden_dim, num_words)
else:
decoder = DecoderRhyme(hidden_dim, num_words, num_target_lengths,
num_rhymes)
if args.train:
weight = torch.ones(num_words)
weight[word2idx_mapping[PAD_TOKEN]] = 0
if use_cuda:
encoder = encoder.cuda()
decoder = decoder.cuda()
weight = weight.cuda()
encoder_optimizer = Adam(encoder.parameters(), lr=0.001)
decoder_optimizer = Adam(decoder.parameters(), lr=0.001)
criterion = nn.CrossEntropyLoss(weight=weight)
np.random.seed(1124)
order = np.arange(len(train_data))
best_loss = 1e10
best_epoch = 0
for e in range(epoch):
#if e - best_epoch > 20: break
np.random.shuffle(order)
shuffled_train_data = train_data[order]
shuffled_x_lengths = input_lengths[order]
shuffled_y_lengths = target_lengths[order]
shuffled_y_rhyme = target_rhymes[order]
train_loss = 0
valid_loss = 0
for b in tqdm(range(int(len(order) // batch_size))):
#print(b, '\r', end='')
batch_x = torch.LongTensor(
shuffled_train_data[b * batch_size:(b + 1) *
batch_size][:, 0].tolist()).t()
batch_y = torch.LongTensor(
shuffled_train_data[b * batch_size:(b + 1) *
batch_size][:, 1].tolist()).t()
batch_x_lengths = shuffled_x_lengths[b * batch_size:(b + 1) *
batch_size]
batch_y_lengths = shuffled_y_lengths[b * batch_size:(b + 1) *
batch_size]
batch_y_rhyme = shuffled_y_rhyme[b * batch_size:(b + 1) *
batch_size]
if use_cuda:
batch_x, batch_y = batch_x.cuda(), batch_y.cuda()
train_loss += train(batch_x, batch_y, batch_y_lengths,
max(batch_y_lengths), batch_y_rhyme,
encoder, decoder, encoder_optimizer,
decoder_optimizer, criterion, use_cuda,
False)
train_loss /= b
'''
for b in range(len(valid_data) // batch_size):
batch_x = torch.LongTensor(valid_data[b*batch_size: (b+1)*batch_size][:, 0].tolist()).t()
batch_y = torch.LongTensor(valid_data[b*batch_size: (b+1)*batch_size][:, 1].tolist()).t()
if use_cuda:
batch_x, batch_y = batch_x.cuda(), batch_y.cuda()
valid_loss += train(batch_x, batch_y, max_seqlen, encoder, decoder, encoder_optimizer, decoder_optimizer, criterion, use_cuda, True)
valid_loss /= b
'''
print(
"epoch {}, train_loss {:.4f}, valid_loss {:.4f}, best_epoch {}, best_loss {:.4f}"
.format(e, train_loss, valid_loss, best_epoch, best_loss))
'''
if valid_loss < best_loss:
best_loss = valid_loss
best_epoch = e
torch.save(encoder.state_dict(), args.encoder_path + '.best')
torch.save(decoder.state_dict(), args.decoder_path + '.best')
'''
torch.save(encoder.state_dict(), args.encoder_path)
torch.save(decoder.state_dict(), args.decoder_path)
print(encoder)
print(decoder)
print("==============")
else:
encoder.load_state_dict(torch.load(
args.encoder_path)) #, map_location=torch.device('cpu')))
decoder.load_state_dict(torch.load(
args.decoder_path)) #, map_location=torch.device('cpu')))
print(encoder)
print(decoder)
predict(encoder, decoder)
def runTest(n_layers, hidden_size, reverse, modelFile, beam_size, input,
corpus):
voc, pairs, valid_pairs, test_pairs = loadPrepareData(corpus)
print('Building encoder and decoder ...')
# aspect
with open(os.path.join(save_dir, '15_aspect.pkl'), 'rb') as fp:
aspect_ids = pickle.load(fp)
aspect_num = 15 # 15 | 20 main aspects and each of them has 100 words
aspect_ids = Variable(
torch.LongTensor(aspect_ids), requires_grad=False
) # convert list into torch Variable, used to index word embedding
# attribute embeddings
attr_size = 64 #
attr_num = 2
with open(os.path.join(save_dir, 'user_item.pkl'), 'rb') as fp:
user_dict, item_dict = pickle.load(fp)
num_user = len(user_dict)
num_item = len(item_dict)
attr_embeddings = []
attr_embeddings.append(nn.Embedding(num_user, attr_size))
attr_embeddings.append(nn.Embedding(num_item, attr_size))
aspect_embeddings = []
aspect_embeddings.append(nn.Embedding(num_user, aspect_num))
aspect_embeddings.append(nn.Embedding(num_item, aspect_num))
if USE_CUDA:
for attr_embedding in attr_embeddings:
attr_embedding = attr_embedding.cuda()
for aspect_embedding in aspect_embeddings:
aspect_embedding = aspect_embedding.cuda()
aspect_ids = aspect_ids.cuda()
encoder1 = AttributeEncoder(attr_size, attr_num, hidden_size,
attr_embeddings, n_layers)
encoder2 = AttributeEncoder(aspect_num, attr_num, hidden_size,
aspect_embeddings, n_layers)
embedding = nn.Embedding(voc.n_words, hidden_size)
encoder3 = EncoderRNN(voc.n_words, hidden_size, embedding, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
attr_size, voc.n_words, aspect_ids, n_layers)
checkpoint = torch.load(modelFile)
encoder1.load_state_dict(checkpoint['en1'])
encoder2.load_state_dict(checkpoint['en2'])
encoder3.load_state_dict(checkpoint['en3'])
decoder.load_state_dict(checkpoint['de'])
# use cuda
if USE_CUDA:
encoder1 = encoder1.cuda()
encoder2 = encoder2.cuda()
encoder3 = encoder3.cuda()
decoder = decoder.cuda()
# train mode set to false, effect only on dropout, batchNorm
encoder1.train(False)
encoder2.train(False)
encoder3.train(False)
decoder.train(False)
#evaluateRandomly(encoder1, encoder2, encoder3, decoder, voc, pairs, reverse, beam_size, 100)
evaluateRandomly(encoder1, encoder2, encoder3, decoder, voc, test_pairs,
reverse, beam_size, len(test_pairs))
def load_network_stageI(self):
from model import STAGE1_G, STAGE1_D
from model import EncoderRNN, LuongAttnDecoderRNN
from model import STAGE1_ImageEncoder, EncodingDiscriminator
netG = STAGE1_G()
netG.apply(weights_init)
#print(netG)
netD = STAGE1_D()
netD.apply(weights_init)
#print(netD)
emb_dim = 300
encoder = EncoderRNN(emb_dim, self.txt_emb, 1, dropout=0.0)
attn_model = 'general'
decoder = LuongAttnDecoderRNN(attn_model,
emb_dim,
len(self.txt_dico.id2word),
self.txt_emb,
n_layers=1,
dropout=0.0)
image_encoder = STAGE1_ImageEncoder()
image_encoder.apply(weights_init)
e_disc = EncodingDiscriminator(emb_dim)
if cfg.NET_G != '':
state_dict = \
torch.load(cfg.NET_G,
map_location=lambda storage, loc: storage)
netG.load_state_dict(state_dict)
print('Load from: ', cfg.NET_G)
if cfg.NET_D != '':
state_dict = \
torch.load(cfg.NET_D,
map_location=lambda storage, loc: storage)
netD.load_state_dict(state_dict)
print('Load from: ', cfg.NET_D)
if cfg.ENCODER != '':
state_dict = \
torch.load(cfg.ENCODER,
map_location=lambda storage, loc: storage)
encoder.load_state_dict(state_dict)
print('Load from: ', cfg.ENCODER)
if cfg.DECODER != '':
state_dict = \
torch.load(cfg.DECODER,
map_location=lambda storage, loc: storage)
decoder.load_state_dict(state_dict)
print('Load from: ', cfg.DECODER)
if cfg.IMAGE_ENCODER != '':
state_dict = \
torch.load(cfg.IMAGE_ENCODER,
map_location=lambda storage, loc: storage)
image_encoder.load_state_dict(state_dict)
print('Load from: ', cfg.IMAGE_ENCODER)
# load classification model and freeze weights
#clf_model = models.alexnet(pretrained=True)
clf_model = models.vgg16(pretrained=True)
for param in clf_model.parameters():
param.requires_grad = False
if cfg.CUDA:
netG.cuda()
netD.cuda()
encoder.cuda()
decoder.cuda()
image_encoder.cuda()
e_disc.cuda()
clf_model.cuda()
# ## finetune model for a bit???
# output_size = 512 * 2 * 2
# num_classes = 200
# clf_model.classifier = nn.Sequential(
# nn.Linear(output_size, 1024, bias=True),
# nn.LeakyReLU(0.2),
# nn.Dropout(0.5),
# nn.Linear(1024, num_classes, bias=True)
# )
# clf_optim = optim.SGD(clf_model.parameters(), lr=1e-2, momentum=0.9)
return netG, netD, encoder, decoder, image_encoder, e_disc, clf_model
def trainIters(corpus,
reverse,
n_epoch,
learning_rate,
batch_size,
n_layers,
hidden_size,
print_every,
loadFilename=None,
attn_model='dot',
decoder_learning_ratio=1.0):
print(
"corpus: {}, reverse={}, n_epoch={}, learning_rate={}, batch_size={}, n_layers={}, hidden_size={}, decoder_learning_ratio={}"
.format(corpus, reverse, n_epoch, learning_rate, batch_size, n_layers,
hidden_size, decoder_learning_ratio))
voc, pairs, valid_pairs, test_pairs = loadPrepareData(corpus)
print('load data...')
path = "data/expansion"
# training data
corpus_name = corpus
training_batches = None
try:
training_batches = torch.load(
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'training_batches'),
batch_size)))
except FileNotFoundError:
print('Training pairs not found, generating ...')
training_batches = batchify(pairs, batch_size, voc, reverse)
print('Complete building training pairs ...')
torch.save(
training_batches,
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'training_batches'),
batch_size)))
# validation/test data
eval_batch_size = 10
try:
val_batches = torch.load(
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'val_batches'),
eval_batch_size)))
except FileNotFoundError:
print('Validation pairs not found, generating ...')
val_batches = batchify(valid_pairs,
eval_batch_size,
voc,
reverse,
evaluation=True)
print('Complete building validation pairs ...')
torch.save(
val_batches,
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'val_batches'),
eval_batch_size)))
try:
test_batches = torch.load(
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'test_batches'),
eval_batch_size)))
except FileNotFoundError:
print('Test pairs not found, generating ...')
test_batches = batchify(test_pairs,
eval_batch_size,
voc,
reverse,
evaluation=True)
print('Complete building test pairs ...')
torch.save(
test_batches,
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'test_batches'),
eval_batch_size)))
# model
checkpoint = None
print('Building encoder and decoder ...')
# aspect
with open(os.path.join(save_dir, '15_aspect.pkl'), 'rb') as fp:
aspect_ids = pickle.load(fp)
aspect_num = 15 # 15 | 20 main aspects and each of them has 100 words
aspect_ids = Variable(
torch.LongTensor(aspect_ids), requires_grad=False
) # convert list into torch Variable, used to index word embedding
# attribute embeddings
attr_size = 64 #
attr_num = 2
print(
"corpus: {}, reverse={}, n_words={}, n_epoch={}, learning_rate={}, batch_size={}, n_layers={}, hidden_size={}, decoder_learning_ratio={}, attr_size={}, aspect_num={}"
.format(corpus, reverse, voc.n_words, n_epoch, learning_rate,
batch_size, n_layers, hidden_size, decoder_learning_ratio,
attr_size, aspect_num))
with open(os.path.join(save_dir, 'user_item.pkl'), 'rb') as fp:
user_dict, item_dict = pickle.load(fp)
num_user = len(user_dict)
num_item = len(item_dict)
attr_embeddings = []
attr_embeddings.append(nn.Embedding(num_user, attr_size))
attr_embeddings.append(nn.Embedding(num_item, attr_size))
aspect_embeddings = []
aspect_embeddings.append(nn.Embedding(num_user, aspect_num))
aspect_embeddings.append(nn.Embedding(num_item, aspect_num))
if USE_CUDA:
for attr_embedding in attr_embeddings:
attr_embedding = attr_embedding.cuda()
for aspect_embedding in aspect_embeddings:
aspect_embedding = aspect_embedding.cuda()
aspect_ids = aspect_ids.cuda()
encoder1 = AttributeEncoder(attr_size, attr_num, hidden_size,
attr_embeddings, n_layers)
encoder2 = AttributeEncoder(aspect_num, attr_num, hidden_size,
aspect_embeddings, n_layers)
embedding = nn.Embedding(voc.n_words, hidden_size)
encoder3 = EncoderRNN(voc.n_words, hidden_size, embedding, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
attr_size, voc.n_words, aspect_ids, n_layers)
if loadFilename:
checkpoint = torch.load(loadFilename)
encoder1.load_state_dict(checkpoint['en1'])
encoder2.load_state_dict(checkpoint['en2'])
encoder3.load_state_dict(checkpoint['en3'])
decoder.load_state_dict(checkpoint['de'])
# use cuda
if USE_CUDA:
encoder1 = encoder1.cuda()
encoder2 = encoder2.cuda()
encoder3 = encoder3.cuda()
decoder = decoder.cuda()
# optimizer
print('Building optimizers ...')
encoder1_optimizer = optim.Adam(encoder1.parameters(), lr=learning_rate)
encoder2_optimizer = optim.Adam(encoder2.parameters(), lr=learning_rate)
encoder3_optimizer = optim.Adam(encoder3.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
if loadFilename:
encoder1_optimizer.load_state_dict(checkpoint['en1_opt'])
encoder2_optimizer.load_state_dict(checkpoint['en2_opt'])
encoder3_optimizer.load_state_dict(checkpoint['en3_opt'])
decoder_optimizer.load_state_dict(checkpoint['de_opt'])
# initialize
print('Initializing ...')
start_epoch = 0
perplexity = []
best_val_loss = None
print_loss = 0
if loadFilename:
start_epoch = checkpoint['epoch'] + 1
perplexity = checkpoint['plt']
for epoch in range(start_epoch, n_epoch):
epoch_start_time = time.time()
# train epoch
encoder1.train()
encoder2.train()
encoder3.train()
decoder.train()
print_loss = 0
start_time = time.time()
for batch, training_batch in enumerate(training_batches):
attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = training_batch
loss = train(attr_input, summary_input, summary_input_lengths,
title_input, title_input_lengths, target_variable,
mask, max_target_len, encoder1, encoder2, encoder3,
decoder, embedding, encoder1_optimizer,
encoder2_optimizer, encoder3_optimizer,
decoder_optimizer, batch_size)
print_loss += loss
perplexity.append(loss)
#print("batch {} loss={}".format(batch, loss))
if batch % print_every == 0 and batch > 0:
cur_loss = print_loss / print_every
elapsed = time.time() - start_time
print(
'| epoch {:3d} | {:5d}/{:5d} batches | lr {:05.5f} | ms/batch {:5.2f} | '
'loss {:5.2f} | ppl {:8.2f}'.format(
epoch, batch, len(training_batches), learning_rate,
elapsed * 1000 / print_every, cur_loss,
math.exp(cur_loss)))
print_loss = 0
start_time = time.time()
# evaluate
val_loss = 0
for val_batch in val_batches:
attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = val_batch
loss = evaluate(attr_input, summary_input, summary_input_lengths,
title_input, title_input_lengths, target_variable,
mask, max_target_len, encoder1, encoder2, encoder3,
decoder, embedding, encoder1_optimizer,
encoder2_optimizer, encoder3_optimizer,
decoder_optimizer, batch_size)
val_loss += loss
val_loss /= len(val_batches)
print('-' * 89)
print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | '
'valid ppl {:8.2f}'.format(epoch,
(time.time() - epoch_start_time),
val_loss, math.exp(val_loss)))
print('-' * 89)
# Save the model if the validation loss is the best we've seen so far.
if not best_val_loss or val_loss < best_val_loss:
directory = os.path.join(save_dir, 'model',
'{}_{}'.format(n_layers, hidden_size))
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'epoch': epoch,
'en1': encoder1.state_dict(),
'en2': encoder2.state_dict(),
'en3': encoder3.state_dict(),
'de': decoder.state_dict(),
'en1_opt': encoder1_optimizer.state_dict(),
'en2_opt': encoder2_optimizer.state_dict(),
'en3_opt': encoder3_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'plt': perplexity
},
os.path.join(
directory, '{}_{}.tar'.format(
epoch,
filename(reverse, 'lexicon_title_expansion_model'))))
best_val_loss = val_loss
# Run on test data.
test_loss = 0
for test_batch in test_batches:
attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = test_batch
loss = evaluate(attr_input, summary_input,
summary_input_lengths, title_input,
title_input_lengths, target_variable, mask,
max_target_len, encoder1, encoder2, encoder3,
decoder, embedding, encoder1_optimizer,
encoder2_optimizer, encoder3_optimizer,
decoder_optimizer, batch_size)
test_loss += loss
test_loss /= len(test_batches)
print('-' * 89)
print('| test loss {:5.2f} | test ppl {:8.2f}'.format(
test_loss, math.exp(test_loss)))
print('-' * 89)
if val_loss > best_val_loss:
break