max_length = data_preprocess.max_length
input_lang, output_lang, pairs = data_preprocess.prepare_data(
'eng', 'fra', True)
print(random.choice(pairs))
helpFn = Helper(max_length)
''' Use pre-trained word embeddings '''
# embedding_src = GetEmbedding(input_lang.word2index, input_lang.word2count, "../Embeddings/GoogleNews/")
# embedding_dest = GetEmbedding(output_lang.word2index, input_lang.word2count, "../Embeddings/GoogleNews/")
# encoder = EncoderRNN(hidden_size, torch.from_numpy(embedding_src.embedding_matrix).type(torch.FloatTensor),
# use_embedding=True, train_embedding=False)
# decoder = DecoderRNN(hidden_size, torch.from_numpy(embedding_dest.embedding_matrix).type(torch.FloatTensor),
# use_embedding=True, train_embedding=False, dropout_p=0.1)
''' Generate and learn embeddings '''
encoder = EncoderRNN(hidden_size, (len(input_lang.word2index), 300),
batch_size)
decoder = DecoderRNN(hidden_size, (len(output_lang.word2index), 300))
if use_cuda:
encoder = encoder.cuda()
decoder = decoder.cuda()
print("Training Network.")
train_network = TrainNetwork(encoder, decoder, output_lang, max_length,
batch_size)
trainIters(train_network, input_lang, output_lang, pairs, max_length,
batch_size)
evaluateRandomly(train_network, input_lang, pairs)
def __init__(self,
src_vocab_size,
tgt_vocab_size,
src_emb_dim,
tgt_emb_dim,
enc_hidden_size,
dec_hidden_size,
context_hidden_size,
batch_size,
image_in_size,
bidirectional_enc=True,
bidirectional_context=False,
num_enc_layers=1,
num_dec_layers=1,
num_context_layers=1,
dropout_enc=0.4,
dropout_dec=0.4,
dropout_context=0.4,
max_decode_len=40,
non_linearity='tanh',
enc_type='GRU',
dec_type='GRU',
context_type='GRU',
use_attention=True,
decode_function='softmax',
sos_id=2,
eos_id=3,
tie_embedding=True,
activation_bridge='Tanh',
num_states=None,
use_kb=False,
kb_size=None,
celeb_vec_size=None):
super(HRED, self).__init__()
self.src_vocab_size = src_vocab_size
self.tgt_vocab_size = tgt_vocab_size
self.src_emb_dim = src_emb_dim
self.tgt_emb_dim = tgt_emb_dim
self.batch_size = batch_size
self.bidirectional_enc = bidirectional_enc
self.bidirectional_context = bidirectional_context
self.num_enc_layers = num_enc_layers
self.num_dec_layers = num_dec_layers
self.num_context_layers = num_context_layers
self.dropout_enc = dropout_enc #dropout prob for encoder
self.dropout_dec = dropout_dec #dropout prob for decoder
self.dropout_context = dropout_context #dropout prob for context
self.non_linearity = non_linearity # default nn.tanh(); nn.relu()
self.enc_type = enc_type
self.dec_type = dec_type
self.context_type = context_type
self.sos_id = sos_id # start token
self.eos_id = eos_id # end token
self.decode_function = decode_function # @TODO: softmax or log softmax
self.max_decode_len = max_decode_len # max timesteps for decoder
self.attention_size = dec_hidden_size # Same as enc/dec hidden size!!
# self.context_hidden_size = context_hidden_size
# self.enc_hidden_size = enc_hidden_size
# All implementations have encoder hidden size halved
self.num_directions = 2 if bidirectional_enc else 1
self.enc_hidden_size = enc_hidden_size // self.num_directions
self.num_directions = 2 if bidirectional_context else 1
self.context_hidden_size = context_hidden_size // self.num_directions
self.dec_hidden_size = dec_hidden_size
self.use_attention = use_attention
self.image_in_size = image_in_size
self.image_out_size = self.dec_hidden_size # Project on same size as enc hidden
self.use_kb = use_kb
self.kb_size = kb_size
self.celeb_vec_size = celeb_vec_size
# Equating to emb_size = tgt_emb_dim for now
# Default to hidden_size = dec_hidden_size for now.
self.kb_emb_size = self.tgt_emb_dim
self.kb_hidden_size = self.dec_hidden_size
self.kb_encoder = KbEncoder(self.kb_size,
self.kb_emb_size,
self.kb_hidden_size,
rnn_type='GRU',
num_layers=1,
batch_first=True,
dropout=0,
bidirectional=False)
# Same for kb and celebs for now.
self.celeb_encoder = KbEncoder(self.celeb_vec_size,
self.kb_emb_size,
self.kb_hidden_size,
rnn_type='GRU',
num_layers=1,
batch_first=True,
dropout=0,
bidirectional=False)
# Initialize encoder
self.encoder = EncoderRNN(self.src_vocab_size,
self.src_emb_dim,
self.enc_hidden_size,
self.enc_type,
self.num_enc_layers,
batch_first=True,
dropout=self.dropout_enc,
bidirectional=self.bidirectional_enc)
# self.image_encoder = ImageEncoder(self.image_in_size, self.image_out_size)
# Initialize bridge layer
self.activation_bridge = activation_bridge
self.bridge = BridgeLayer(self.enc_hidden_size, self.dec_hidden_size,
self.activation_bridge)
# Initialize context encoder
self.context_input_size = enc_hidden_size #self.image_out_size + enc_hidden_size # image+text
self.context_encoder = ContextRNN(
self.context_input_size,
self.context_hidden_size,
self.context_type,
self.num_context_layers,
batch_first=True,
dropout=self.dropout_context,
bidirectional=self.bidirectional_context)
# Initialize RNN decoder
self.decoder = DecoderRNN(self.tgt_vocab_size,
self.tgt_emb_dim,
self.dec_hidden_size,
self.dec_type,
self.num_dec_layers,
self.max_decode_len,
self.dropout_dec,
batch_first=True,
use_attention=self.use_attention,
attn_size=self.attention_size,
sos_id=self.sos_id,
eos_id=self.eos_id,
use_input_feed=True,
use_kb=self.use_kb,
kb_size=self.kb_hidden_size,
celeb_vec_size=self.kb_hidden_size)
if tie_embedding:
self.decoder.embedding = self.encoder.embedding
# Initialize parameters
self.init_params()
#Encoder
#-------------------------------------------------------------------------------------
#####################################################################################
from encoderRNN import EncoderRNN
vocab_size = 100
input_seq = Variable(torch.randperm(vocab_size).view(10,
10)) # (batch,seq_len)
print input_seq.size()
seq_length = [10, 9, 8, 7, 6, 5, 4, 3, 2, 1] # This works
# seq_length = torch.LongTensor([10,9,8,7,6,5,4,3,2,1]) # This doesn't
encoder = EncoderRNN(100,
6,
2,
'GRU',
1,
batch_first=True,
dropout=0,
bidirectional=True)
for param in encoder.parameters():
param.data.uniform_(-1, 1)
output, hidden = encoder(input_seq, seq_length, hidden=None)
print output.size()
print hidden.size()
print output
print hidden
print type(output)
a = (Dec_in[batch_size*i:batch_size*(i+1)])
# Pad sequence
dec_in_.append( torch.nn.utils.rnn.pad_sequence(a, batch_first=True) )
# Prepare [Encoder input,Decoder input,Ideal decoder output] set
batch_in_out_pairs =[] # may ignore some last couplets
for i in range(0, len(in_)):
batch_in_out_pairs.append((in_[i].squeeze(),dec_in_[i].squeeze(), out_[i].squeeze()))
print("Number of couplet batches :",len(batch_in_out_pairs))
### Starting Training ###
print("### Creating models ###")
# Define encoder and decoder
input_size = vocab_size
output_size = vocab_size
encoder = EncoderRNN(input_size, hidden_size)
attn_decoder = AttnDecoderRNN_II(hidden_size, output_size,dropout_p)
# Define the device
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
encoder.to(device)
attn_decoder.to(device)
# Define the optimizer
encoder_optimizer = optim.Adam(encoder.parameters(), lr=learning_rate)
decoder_optimizer = optim.Adam(attn_decoder.parameters(), lr=learning_rate)
lossesPLL_train = []
lossesPLL_valid = []
times = []
print("### Starting training ###")
return decoded_words, decoder_attentions[:di + 1]
def evaluateRandomly(encoder, decoder, n=10):
for i in range(n):
pair = random.choice(pairs)
print('>', pair[0])
print('=', pair[1])
output_words, attentions = evaluate(encoder, decoder, pair[0])
output_sentence = ' '.join(output_words)
print('<', output_sentence)
print('')
hidden_size = 256
encoder1 = EncoderRNN(input_lang.n_words, hidden_size).to(device)
attn_decoder1 = AttnDecoderRNN(hidden_size, output_lang.n_words,
dropout_p=0.1).to(device)
# trainIters(encoder1, attn_decoder1, 75000, print_every=5000)
# 此处load
output_words, attentions = evaluate(encoder1, attn_decoder1,
"je suis trop froid .")
plt.matshow(attentions.numpy())
def showAttention(input_sentence, output_words, attentions):
# Set up figure with colorbar
fig = plt.figure()
ax = fig.add_subplot(111)
# embedding_src = GetEmbedding(input_lang.word2index, input_lang.word2count, "../Embeddings/GoogleNews/")
embedding_dest = GetEmbedding(output_lang.word2index,
input_lang.word2count,
"../Embeddings/GoogleNews/")
hidden_size = 256
# encoder = EncoderRNN(hidden_size, torch.from_numpy(embedding_src.embedding_matrix).type(torch.FloatTensor),
# use_embedding=True, train_embedding=False)
decoder = DecoderRNN(hidden_size,
data_preprocess.max_length,
torch.from_numpy(
embedding_dest.embedding_matrix).type(
torch.FloatTensor),
use_embedding=True,
train_embedding=False,
dropout_p=0.1)
encoder = EncoderRNN(hidden_size, (len(input_lang.word2index) + 1, 300))
# decoder = DecoderRNN(hidden_size, data_preprocess.max_length, (len(output_lang.word2index) + 1, 300))
if use_cuda:
encoder = encoder.cuda()
decoder = decoder.cuda()
print("Training Network.")
train_network = TrainNetwork(encoder, decoder, output_lang,
data_preprocess.max_length)
trainIters(train_network, input_lang, output_lang, pairs)
evaluateRandomly(train_network, input_lang, pairs)
data_preprocess = DataPreprocess(
"./Datasets/Neural-Dialogue-Generation/data/")
max_length = data_preprocess.max_length
in_seq = data_preprocess.x_train
out_seq = data_preprocess.y_train
lengths = data_preprocess.lengths_train
speakers = data_preprocess.speaker_list_train
addressees = data_preprocess.addressee_list_train
index2word = data_preprocess.index2word
vocab_size = data_preprocess.vocab_size
personas = len(data_preprocess.people) + 1
helpFn = Helper(max_length)
encoder = EncoderRNN(hidden_size, (vocab_size, 300), batch_size)
decoder = DecoderRNN(hidden_size, (vocab_size, 300), (personas, 300))
if use_cuda:
encoder = encoder.cuda()
decoder = decoder.cuda()
print("Training Network.")
train_network = TrainNetwork(encoder, decoder, index2word, max_length,
batch_size)
trainIters(train_network, in_seq, out_seq, [speakers, addressees], lengths,
max_length, batch_size)
evaluateRandomly(train_network, in_seq, out_seq, [speakers, addressees],
lengths, index2word)