def runTest(n_layers, hidden_size, reverse, modelFile, beam_size, inp, corpus):
torch.set_grad_enabled(False)
voc, pairs = loadPrepareData(corpus)
embedding = nn.Embedding(voc.num_words, hidden_size)
encoder = EncoderRNN(hidden_size, embedding, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
voc.num_words, n_layers)
checkpoint = torch.load(modelFile,
map_location=lambda storage, loc: storage)
encoder.load_state_dict(checkpoint['en'])
decoder.load_state_dict(checkpoint['de'])
# train mode set to false, effect only on dropout, batchNorm
encoder.train(False)
decoder.train(False)
encoder = encoder.to(device)
decoder = decoder.to(device)
if inp:
evaluateInput(encoder, decoder, voc, beam_size)
else:
evaluateRandomly(encoder, decoder, voc, pairs, reverse, beam_size, 20)
def runTest(args, n_layers, hidden_size, reverse, modelFile, beam_size,
batch_size, input, corpus):
data, length = loadPrepareData(args)
voc = data.voc
print('load data...')
user_length, item_length = length #, user_length2, item_length2 = length
# train_batches = batchify(data.train, data.user_text, user_length, data.item_text, item_length, batch_size)
# val_batches = batchify(data.dev, data.user_text, user_length, data.item_text, item_length, batch_size)
test_batches = batchify(data.test, data.user_text, user_length,
data.item_text, item_length, batch_size)
print('Building encoder and decoder ...')
embedding = nn.Embedding(data.voc.n_words, hidden_size)
encoderU = EncoderRNNlinear(data.voc.n_words, hidden_size, embedding,
data.dmax, n_layers)
encoderB = EncoderRNNlinear(data.voc.n_words, hidden_size, embedding,
data.dmax, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
data.voc.n_words, n_layers)
# load model
checkpoint = torch.load(modelFile)
encoderU.load_state_dict(checkpoint['enU'])
encoderB.load_state_dict(checkpoint['enB'])
decoder.load_state_dict(checkpoint['de'])
# train mode set to false, effect only on dropout, batchNorm
encoderU.train(False)
encoderB.train(False)
decoder.train(False)
if USE_CUDA:
encoderU = encoderU.cuda()
encoderB = encoderB.cuda()
decoder = decoder.cuda()
if not args.sample:
# evaluate on test
# for test_batch in tqdm(test_batches):
for test_i, test_batch in enumerate(test_batches):
if test_i > 1:
break
input_index, input_variable, lengths, target_variable, mask, max_target_len = test_batch
user_input_variable, business_input_variable = input_variable
user_lengths, business_lengths = lengths
# evaluate on train
evaluateRandomly(encoderU, encoderB, decoder, voc, \
input_index, user_input_variable, business_input_variable, \
user_lengths, business_lengths, \
target_variable, mask, max_target_len, reverse, beam_size)
else:
# evaluate using sample
sample(encoderU, encoderB, decoder, voc, test_batches, reverse)
def trainIters(attn_model, hidden_size,encoder_n_layers, decoder_n_layers, save_dir, n_iteration, batch_size, \
learning_rate, decoder_learning_ratio, print_every, save_every, clip, dropout, \
corpus_name, datafile, modelFile=None, need_trim=True):
# load train data
voc, pairs = loadPrepareData(datafile)
if need_trim:
# Trim voc and pairs
pairs = trimRareWords(voc, pairs, MIN_COUNT)
# Load batches for each iteration
training_batches = [
batch2TrainData(voc, [random.choice(pairs) for _ in range(batch_size)])
for _ in range(n_iteration)
]
if modelFile:
# If loading on same machine the model was trained on
checkpoint = torch.load(modelFile)
# If loading a model trained on GPU to CPU
# checkpoint = torch.load(loadFilename, map_location=torch.device('cpu'))
encoder_sd = checkpoint['en']
decoder_sd = checkpoint['de']
encoder_optimizer_sd = checkpoint['en_opt']
decoder_optimizer_sd = checkpoint['de_opt']
embedding_sd = checkpoint['embedding']
embedding = nn.Embedding(voc.num_words, hidden_size)
if modelFile:
embedding.load_state_dict(embedding_sd)
# Initialize encoder & decoder models
encoder = EncoderRNN(hidden_size, embedding, encoder_n_layers, dropout)
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
voc.num_words, decoder_n_layers, dropout)
# get model params
if modelFile:
encoder.load_state_dict(encoder_sd)
decoder.load_state_dict(decoder_sd)
print('Building optimizers ...')
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
if modelFile:
encoder_optimizer.load_state_dict(encoder_optimizer_sd)
decoder_optimizer.load_state_dict(decoder_optimizer_sd)
# Initializations
print('Initializing ...')
start_iteration = 1
print_loss = 0
if modelFile:
start_iteration = checkpoint['iteration'] + 1
# Training loop
print("Training...")
encoder.train()
decoder.train()
for iteration in range(start_iteration, n_iteration + 1):
training_batch = training_batches[iteration - 1]
# Extract fields from batch
input_variable, lengths, target_variable, mask, max_target_len = training_batch
# Run a training iteration with batch
loss = train(input_variable, lengths, target_variable, mask,
max_target_len, encoder, decoder, encoder_optimizer,
decoder_optimizer, batch_size, clip)
print_loss += loss
# Print progress
if iteration % print_every == 0:
print_loss_avg = print_loss / print_every
print("Iteration: {}; Percent complete: {:.1f}%; Average loss: {:.4f}".format(iteration, \
iteration / n_iteration * 100, print_loss_avg))
print_loss = 0
# Save checkpoint
if (iteration % save_every == 0):
directory = os.path.join(save_dir, "model", '{}-{}_{}'.format(encoder_n_layers, \
decoder_n_layers, hidden_size))
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'iteration': iteration,
'en': encoder.state_dict(),
'de': decoder.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'embedding': embedding.state_dict()
},
os.path.join(directory,
'{}_{}.tar'.format(iteration, 'checkpoint')))
def train(**kwargs):
opt = Config()
for k, v in kwargs.items(): #设置参数
setattr(opt, k, v)
# 数据
dataloader = get_dataloader(opt)
_data = dataloader.dataset._data
word2ix = _data['word2ix']
sos = word2ix.get(_data.get('sos'))
voc_length = len(word2ix)
#定义模型
encoder = EncoderRNN(opt, voc_length)
decoder = LuongAttnDecoderRNN(opt, voc_length)
#加载断点,从上次结束地方开始
if opt.model_ckpt:
checkpoint = torch.load(opt.model_ckpt)
encoder.load_state_dict(checkpoint['en'])
decoder.load_state_dict(checkpoint['de'])
#切换模式
encoder = encoder.to(opt.device)
decoder = decoder.to(opt.device)
encoder.train()
decoder.train()
#定义优化器(注意与encoder.to(device)前后不要反)
encoder_optimizer = torch.optim.Adam(encoder.parameters(),
lr=opt.learning_rate)
decoder_optimizer = torch.optim.Adam(decoder.parameters(),
lr=opt.learning_rate *
opt.decoder_learning_ratio)
if opt.model_ckpt:
encoder_optimizer.load_state_dict(checkpoint['en_opt'])
decoder_optimizer.load_state_dict(checkpoint['de_opt'])
#定义打印loss的变量
print_loss = 0
for epoch in range(opt.epoch):
for ii, data in enumerate(dataloader):
#取一个batch训练
loss = train_by_batch(sos, opt, data, encoder_optimizer,
decoder_optimizer, encoder, decoder)
print_loss += loss
#打印损失
if ii % opt.print_every == 0:
print_loss_avg = print_loss / opt.print_every
print(
"Epoch: {}; Epoch Percent complete: {:.1f}%; Average loss: {:.4f}"
.format(epoch, epoch / opt.epoch * 100, print_loss_avg))
print_loss = 0
# 保存checkpoint
if epoch % opt.save_every == 0:
checkpoint_path = '{prefix}_{time}'.format(
prefix=opt.prefix, time=time.strftime('%m%d_%H%M'))
torch.save(
{
'en': encoder.state_dict(),
'de': decoder.state_dict(),
'en_opt': encoder_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
}, checkpoint_path)
decoder = decoder.to(device)
print('Models built and ready to go!')
# step8: do train
# 配置训练/优化
clip = 50.0
teacher_forcing_ratio = 1.0
learning_rate = 0.0001
decoder_learning_ratio = 5.0
n_iteration = 4000
print_every = 1
save_every = 500
# 确保dropout layers在训练模型中
encoder.train()
decoder.train()
# 初始化优化器
print('Building optimizers ...')
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
if loadFilename:
encoder_optimizer.load_state_dict(encoder_optimizer_sd)
decoder_optimizer.load_state_dict(decoder_optimizer_sd)
# 运行训练迭代
print("Starting Training!")
trainIters(model_name, voc, pairs, encoder, decoder, encoder_optimizer,
decoder_optimizer, embedding, encoder_n_layers, decoder_n_layers,
save_dir, n_iteration, batch_size, print_every, save_every, clip,
def runTest(n_layers, hidden_size, reverse, modelFile, beam_size, input,
corpus):
voc, pairs, valid_pairs, test_pairs = loadPrepareData(corpus)
print('Building encoder and decoder ...')
# aspect
with open(os.path.join(save_dir, '15_aspect.pkl'), 'rb') as fp:
aspect_ids = pickle.load(fp)
aspect_num = 15 # 15 | 20 main aspects and each of them has 100 words
aspect_ids = Variable(
torch.LongTensor(aspect_ids), requires_grad=False
) # convert list into torch Variable, used to index word embedding
# attribute embeddings
attr_size = 64 #
attr_num = 2
with open(os.path.join(save_dir, 'user_item.pkl'), 'rb') as fp:
user_dict, item_dict = pickle.load(fp)
num_user = len(user_dict)
num_item = len(item_dict)
attr_embeddings = []
attr_embeddings.append(nn.Embedding(num_user, attr_size))
attr_embeddings.append(nn.Embedding(num_item, attr_size))
aspect_embeddings = []
aspect_embeddings.append(nn.Embedding(num_user, aspect_num))
aspect_embeddings.append(nn.Embedding(num_item, aspect_num))
if USE_CUDA:
for attr_embedding in attr_embeddings:
attr_embedding = attr_embedding.cuda()
for aspect_embedding in aspect_embeddings:
aspect_embedding = aspect_embedding.cuda()
aspect_ids = aspect_ids.cuda()
encoder1 = AttributeEncoder(attr_size, attr_num, hidden_size,
attr_embeddings, n_layers)
encoder2 = AttributeEncoder(aspect_num, attr_num, hidden_size,
aspect_embeddings, n_layers)
embedding = nn.Embedding(voc.n_words, hidden_size)
encoder3 = EncoderRNN(voc.n_words, hidden_size, embedding, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
attr_size, voc.n_words, aspect_ids, n_layers)
checkpoint = torch.load(modelFile)
encoder1.load_state_dict(checkpoint['en1'])
encoder2.load_state_dict(checkpoint['en2'])
encoder3.load_state_dict(checkpoint['en3'])
decoder.load_state_dict(checkpoint['de'])
# use cuda
if USE_CUDA:
encoder1 = encoder1.cuda()
encoder2 = encoder2.cuda()
encoder3 = encoder3.cuda()
decoder = decoder.cuda()
# train mode set to false, effect only on dropout, batchNorm
encoder1.train(False)
encoder2.train(False)
encoder3.train(False)
decoder.train(False)
#evaluateRandomly(encoder1, encoder2, encoder3, decoder, voc, pairs, reverse, beam_size, 100)
evaluateRandomly(encoder1, encoder2, encoder3, decoder, voc, test_pairs,
reverse, beam_size, len(test_pairs))
def trainIters(corpus,
reverse,
n_epoch,
learning_rate,
batch_size,
n_layers,
hidden_size,
print_every,
loadFilename=None,
attn_model='dot',
decoder_learning_ratio=1.0):
print(
"corpus: {}, reverse={}, n_epoch={}, learning_rate={}, batch_size={}, n_layers={}, hidden_size={}, decoder_learning_ratio={}"
.format(corpus, reverse, n_epoch, learning_rate, batch_size, n_layers,
hidden_size, decoder_learning_ratio))
voc, pairs, valid_pairs, test_pairs = loadPrepareData(corpus)
print('load data...')
path = "data/expansion"
# training data
corpus_name = corpus
training_batches = None
try:
training_batches = torch.load(
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'training_batches'),
batch_size)))
except FileNotFoundError:
print('Training pairs not found, generating ...')
training_batches = batchify(pairs, batch_size, voc, reverse)
print('Complete building training pairs ...')
torch.save(
training_batches,
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'training_batches'),
batch_size)))
# validation/test data
eval_batch_size = 10
try:
val_batches = torch.load(
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'val_batches'),
eval_batch_size)))
except FileNotFoundError:
print('Validation pairs not found, generating ...')
val_batches = batchify(valid_pairs,
eval_batch_size,
voc,
reverse,
evaluation=True)
print('Complete building validation pairs ...')
torch.save(
val_batches,
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'val_batches'),
eval_batch_size)))
try:
test_batches = torch.load(
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'test_batches'),
eval_batch_size)))
except FileNotFoundError:
print('Test pairs not found, generating ...')
test_batches = batchify(test_pairs,
eval_batch_size,
voc,
reverse,
evaluation=True)
print('Complete building test pairs ...')
torch.save(
test_batches,
os.path.join(
save_dir, path,
'{}_{}.tar'.format(filename(reverse, 'test_batches'),
eval_batch_size)))
# model
checkpoint = None
print('Building encoder and decoder ...')
# aspect
with open(os.path.join(save_dir, '15_aspect.pkl'), 'rb') as fp:
aspect_ids = pickle.load(fp)
aspect_num = 15 # 15 | 20 main aspects and each of them has 100 words
aspect_ids = Variable(
torch.LongTensor(aspect_ids), requires_grad=False
) # convert list into torch Variable, used to index word embedding
# attribute embeddings
attr_size = 64 #
attr_num = 2
print(
"corpus: {}, reverse={}, n_words={}, n_epoch={}, learning_rate={}, batch_size={}, n_layers={}, hidden_size={}, decoder_learning_ratio={}, attr_size={}, aspect_num={}"
.format(corpus, reverse, voc.n_words, n_epoch, learning_rate,
batch_size, n_layers, hidden_size, decoder_learning_ratio,
attr_size, aspect_num))
with open(os.path.join(save_dir, 'user_item.pkl'), 'rb') as fp:
user_dict, item_dict = pickle.load(fp)
num_user = len(user_dict)
num_item = len(item_dict)
attr_embeddings = []
attr_embeddings.append(nn.Embedding(num_user, attr_size))
attr_embeddings.append(nn.Embedding(num_item, attr_size))
aspect_embeddings = []
aspect_embeddings.append(nn.Embedding(num_user, aspect_num))
aspect_embeddings.append(nn.Embedding(num_item, aspect_num))
if USE_CUDA:
for attr_embedding in attr_embeddings:
attr_embedding = attr_embedding.cuda()
for aspect_embedding in aspect_embeddings:
aspect_embedding = aspect_embedding.cuda()
aspect_ids = aspect_ids.cuda()
encoder1 = AttributeEncoder(attr_size, attr_num, hidden_size,
attr_embeddings, n_layers)
encoder2 = AttributeEncoder(aspect_num, attr_num, hidden_size,
aspect_embeddings, n_layers)
embedding = nn.Embedding(voc.n_words, hidden_size)
encoder3 = EncoderRNN(voc.n_words, hidden_size, embedding, n_layers)
attn_model = 'dot'
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
attr_size, voc.n_words, aspect_ids, n_layers)
if loadFilename:
checkpoint = torch.load(loadFilename)
encoder1.load_state_dict(checkpoint['en1'])
encoder2.load_state_dict(checkpoint['en2'])
encoder3.load_state_dict(checkpoint['en3'])
decoder.load_state_dict(checkpoint['de'])
# use cuda
if USE_CUDA:
encoder1 = encoder1.cuda()
encoder2 = encoder2.cuda()
encoder3 = encoder3.cuda()
decoder = decoder.cuda()
# optimizer
print('Building optimizers ...')
encoder1_optimizer = optim.Adam(encoder1.parameters(), lr=learning_rate)
encoder2_optimizer = optim.Adam(encoder2.parameters(), lr=learning_rate)
encoder3_optimizer = optim.Adam(encoder3.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
if loadFilename:
encoder1_optimizer.load_state_dict(checkpoint['en1_opt'])
encoder2_optimizer.load_state_dict(checkpoint['en2_opt'])
encoder3_optimizer.load_state_dict(checkpoint['en3_opt'])
decoder_optimizer.load_state_dict(checkpoint['de_opt'])
# initialize
print('Initializing ...')
start_epoch = 0
perplexity = []
best_val_loss = None
print_loss = 0
if loadFilename:
start_epoch = checkpoint['epoch'] + 1
perplexity = checkpoint['plt']
for epoch in range(start_epoch, n_epoch):
epoch_start_time = time.time()
# train epoch
encoder1.train()
encoder2.train()
encoder3.train()
decoder.train()
print_loss = 0
start_time = time.time()
for batch, training_batch in enumerate(training_batches):
attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = training_batch
loss = train(attr_input, summary_input, summary_input_lengths,
title_input, title_input_lengths, target_variable,
mask, max_target_len, encoder1, encoder2, encoder3,
decoder, embedding, encoder1_optimizer,
encoder2_optimizer, encoder3_optimizer,
decoder_optimizer, batch_size)
print_loss += loss
perplexity.append(loss)
#print("batch {} loss={}".format(batch, loss))
if batch % print_every == 0 and batch > 0:
cur_loss = print_loss / print_every
elapsed = time.time() - start_time
print(
'| epoch {:3d} | {:5d}/{:5d} batches | lr {:05.5f} | ms/batch {:5.2f} | '
'loss {:5.2f} | ppl {:8.2f}'.format(
epoch, batch, len(training_batches), learning_rate,
elapsed * 1000 / print_every, cur_loss,
math.exp(cur_loss)))
print_loss = 0
start_time = time.time()
# evaluate
val_loss = 0
for val_batch in val_batches:
attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = val_batch
loss = evaluate(attr_input, summary_input, summary_input_lengths,
title_input, title_input_lengths, target_variable,
mask, max_target_len, encoder1, encoder2, encoder3,
decoder, embedding, encoder1_optimizer,
encoder2_optimizer, encoder3_optimizer,
decoder_optimizer, batch_size)
val_loss += loss
val_loss /= len(val_batches)
print('-' * 89)
print('| end of epoch {:3d} | time: {:5.2f}s | valid loss {:5.2f} | '
'valid ppl {:8.2f}'.format(epoch,
(time.time() - epoch_start_time),
val_loss, math.exp(val_loss)))
print('-' * 89)
# Save the model if the validation loss is the best we've seen so far.
if not best_val_loss or val_loss < best_val_loss:
directory = os.path.join(save_dir, 'model',
'{}_{}'.format(n_layers, hidden_size))
if not os.path.exists(directory):
os.makedirs(directory)
torch.save(
{
'epoch': epoch,
'en1': encoder1.state_dict(),
'en2': encoder2.state_dict(),
'en3': encoder3.state_dict(),
'de': decoder.state_dict(),
'en1_opt': encoder1_optimizer.state_dict(),
'en2_opt': encoder2_optimizer.state_dict(),
'en3_opt': encoder3_optimizer.state_dict(),
'de_opt': decoder_optimizer.state_dict(),
'loss': loss,
'plt': perplexity
},
os.path.join(
directory, '{}_{}.tar'.format(
epoch,
filename(reverse, 'lexicon_title_expansion_model'))))
best_val_loss = val_loss
# Run on test data.
test_loss = 0
for test_batch in test_batches:
attr_input, summary_input, summary_input_lengths, title_input, title_input_lengths, target_variable, mask, max_target_len = test_batch
loss = evaluate(attr_input, summary_input,
summary_input_lengths, title_input,
title_input_lengths, target_variable, mask,
max_target_len, encoder1, encoder2, encoder3,
decoder, embedding, encoder1_optimizer,
encoder2_optimizer, encoder3_optimizer,
decoder_optimizer, batch_size)
test_loss += loss
test_loss /= len(test_batches)
print('-' * 89)
print('| test loss {:5.2f} | test ppl {:8.2f}'.format(
test_loss, math.exp(test_loss)))
print('-' * 89)
if val_loss > best_val_loss:
break
def main():
USE_CUDA = torch.cuda.is_available()
device = torch.device("cuda" if USE_CUDA else "cpu")
# load data
corpus_name = "cornell movie-dialogs corpus"
corpus = os.path.join("data", corpus_name)
datafile = os.path.join(corpus, "formatted_movie_lines.txt")
voc, pairs = loadPrepareData(corpus_name, datafile)
# Trim voc and pairs
pairs = trimRareWords(voc, pairs, MIN_COUNT)
# Configure models
model_name = 'cb_model'
attn_model = 'dot'
# attn_model = 'general'
# attn_model = 'concat'
hidden_size = 500
encoder_n_layers = 2
decoder_n_layers = 2
dropout = 0.1
batch_size = 64
# Set checkpoint to load from; set to None if starting from scratch
loadFilename = None
# checkpoint_iter = 4000
# loadFilename = os.path.join(save_dir, model_name, corpus_name,
# '{}-{}_{}'.format(encoder_n_layers, decoder_n_layers, hidden_size),
# '{}_checkpoint.tar'.format(checkpoint_iter))
# Load model if a loadFilename is provided
checkpoint = None
if loadFilename:
# If loading on same machine the model was trained on
checkpoint = torch.load(loadFilename)
# If loading a model trained on GPU to CPU
# checkpoint = torch.load(loadFilename, map_location=torch.device('cpu'))
encoder_sd = checkpoint['en']
decoder_sd = checkpoint['de']
encoder_optimizer_sd = checkpoint['en_opt']
decoder_optimizer_sd = checkpoint['de_opt']
embedding_sd = checkpoint['embedding']
voc.__dict__ = checkpoint['voc_dict']
print('Building encoder and decoder ...')
# Initialize word embeddings
embedding = nn.Embedding(voc.num_words, hidden_size)
if loadFilename:
embedding.load_state_dict(embedding_sd)
# Initialize encoder & decoder models
encoder = EncoderRNN(hidden_size, embedding, encoder_n_layers, dropout)
decoder = LuongAttnDecoderRNN(attn_model, embedding, hidden_size,
voc.num_words, decoder_n_layers, dropout)
if loadFilename:
encoder.load_state_dict(encoder_sd)
decoder.load_state_dict(decoder_sd)
# Use appropriate device
encoder = encoder.to(device)
decoder = decoder.to(device)
print('Models built and ready to go!')
# Configure training/optimization
clip = 50.0
teacher_forcing_ratio = 1.0
learning_rate = 0.0001
decoder_learning_ratio = 5.0
n_iteration = 4000
print_every = 1
save_every = 500
# Ensure dropout layers are in train mode
encoder.train()
decoder.train()
# Initialize optimizers
print('Building optimizers ...')
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=learning_rate * decoder_learning_ratio)
if loadFilename:
encoder_optimizer.load_state_dict(encoder_optimizer_sd)
decoder_optimizer.load_state_dict(decoder_optimizer_sd)
# Run training iterations
print("Starting Training!")
save_dir = os.path.join("data", "save")
trainIters(model_name, voc, pairs, encoder, decoder, encoder_optimizer,
decoder_optimizer, embedding, encoder_n_layers,
decoder_n_layers, save_dir, n_iteration, batch_size,
print_every, save_every, clip, corpus_name, checkpoint,
hidden_size, teacher_forcing_ratio, device)
def main():
phase = {"train": {"pairs": []}, "test": {"pairs": []}}
if run_mode == 'train':
with open(datafiles["qr_train"], "r") as file_obj:
for line in file_obj:
phase["train"]["pairs"].append(line.split("\n")[0].split("\t"))
with open(f"{os.path.join(split_path, 'voc.pickle')}", "rb") as f:
phase["train"]["voc"] = pickle.load(f)
# Shuffle both sets ONCE before the entire training
random.seed(1) # seed can be any number
random.shuffle(phase["train"]["pairs"])
print('Building training set encoder and decoder ...')
# Initialize word embeddings for both encoder and decoder
embedding = nn.Embedding(phase["train"]["voc"].num_words,
HIDDEN_SIZE).to(device)
# Initialize encoder & decoder models
encoder = EncoderRNN(HIDDEN_SIZE,
embedding,
ENCODER_N_LAYERS,
DROPOUT,
gate=encoder_name,
bidirectional=BIDIRECTION)
decoder = LuongAttnDecoderRNN(attn_model,
embedding,
HIDDEN_SIZE,
phase["train"]["voc"].num_words,
DECODER_N_LAYERS,
DROPOUT,
gate=decoder_name)
# Use appropriate device
encoder = encoder.to(device)
decoder = decoder.to(device)
encoder.train()
decoder.train()
print('Models built and ready to go!')
# Initialize optimizers
print('Building optimizers ...')
if args.get('optimizer') == "ADAM":
encoder_optimizer = optim.Adam(encoder.parameters(),
lr=LR,
weight_decay=WD)
decoder_optimizer = optim.Adam(decoder.parameters(),
lr=LR,
weight_decay=WD)
elif args.get('optimizer') == "SGD":
encoder_optimizer = optim.SGD(encoder.parameters(), lr=LR)
decoder_optimizer = optim.SGD(decoder.parameters(), lr=LR)
else:
raise ValueError(
"Wrong optimizer type has been given as an argument.")
# If you have cuda, configure cuda to call
for optimizer in [encoder_optimizer, decoder_optimizer]:
for state in optimizer.state.values():
for k, v in state.items():
if isinstance(v, torch.Tensor):
state[k] = v.cuda()
print("Starting Training!")
save_model = run(encoder,
decoder,
encoder_optimizer,
decoder_optimizer,
EPOCH_NUM,
BATCH_SIZE,
CLIP,
phase,
evaluation=True)
if save_model:
try:
save_seq2seq(encoder, decoder, encoder_name, decoder_name,
encoder_optimizer, decoder_optimizer,
phase["train"]["losses"], phase["train"]["bleu"],
phase["train"]["voc"], embedding, DROPOUT, CLIP,
WD)
print("Model has been saved successfully.")
except Exception as error:
print("Saving the model has caused an exception:", error)
write_results("loss", "train", encoder, encoder_name, decoder_name,
DROPOUT, CLIP, WD, phase["train"]["losses"])
write_results("bleu", "train", encoder, encoder_name, decoder_name,
DROPOUT, CLIP, WD, phase["train"]["bleu"])
else:
# Loading basic objects needed for all 3 of validation, testing and chatting
checkpoint = torch.load(args.get('model_path'))
embedding = load_embedding(checkpoint, HIDDEN_SIZE)
encoder = load_encoder(checkpoint, EncoderRNN, HIDDEN_SIZE, embedding,
ENCODER_N_LAYERS, DROPOUT, encoder_name,
BIDIRECTION)
voc = load_voc(checkpoint)
decoder = load_decoder(checkpoint, LuongAttnDecoderRNN, attn_model,
embedding, HIDDEN_SIZE, voc.num_words,
DECODER_N_LAYERS, DROPOUT, decoder_name)
encoder = encoder.to(device)
decoder = decoder.to(device)
if run_mode == "test":
with open(datafiles["qr_train"], "r") as file_obj:
for line in file_obj:
phase["train"]["pairs"].append(
line.split("\n")[0].split("\t"))
with open(datafiles["qr_test"], "r") as file_obj:
for line in file_obj:
phase["test"]["pairs"].append(
line.split("\n")[0].split("\t"))
with open(f"{os.path.join(split_path, 'voc.pickle')}", "rb") as f:
phase["train"]["voc"] = pickle.load(f)
_ = run(encoder,
decoder,
None,
None,
EPOCH_NUM,
BATCH_SIZE,
CLIP,
phase,
evaluation=True)
write_results("loss", "train", encoder, encoder_name, decoder_name,
DROPOUT, CLIP, WD, phase["train"]["losses"])
write_results("bleu", "train", encoder, encoder_name, decoder_name,
DROPOUT, CLIP, WD, phase["train"]["bleu"])
write_results("loss", "test", encoder, encoder_name, decoder_name,
DROPOUT, CLIP, WD, phase["test"]["losses"])
write_results("bleu", "test", encoder, encoder_name, decoder_name,
DROPOUT, CLIP, WD, phase["test"]["bleu"])
elif run_mode == "chat":
# Initialize search module
searcher = GreedySearchDecoder(encoder, decoder)
chat(searcher, voc)
else:
raise ValueError(
"Wrong run_mode has been given, options: ['train', 'test', 'chat']"
)
