def test_accumulator():
with tf.Graph().as_default():
iw = tf.constant([[1, 2, 3, 0, 0], [2, 1, 3, 1, 9], [1, 0, 0, 0, 0]],
dtype=tf.float32)
iw = tf.expand_dims(iw, 2)
iw_len = tf.constant([3, 5, 1])
dw = tf.constant([[1, 3, 3, 4, 0], [3, 4, 3, 4, 5], [3, 0, 0, 0, 0]],
dtype=tf.float32)
dw = tf.expand_dims(dw, 2)
dw_len = tf.constant([4, 5, 1])
edit_vector = ev.accumulator_encoder(iw, dw, iw_len, dw_len, 10, 100,
0.1, 14, 0.8)
with tf.Session() as sess:
sess.run([
tf.global_variables_initializer(),
tf.local_variables_initializer(),
tf.tables_initializer()
])
o = sess.run(edit_vector)
assert o.shape == (3, 10)
def editor_train(base_words,
source_words,
target_words,
insert_words,
delete_words,
embed_matrix,
vocab_table,
hidden_dim,
agenda_dim,
edit_dim,
num_encoder_layers,
num_decoder_layers,
attn_dim,
beam_width,
max_sent_length,
dropout_keep,
lamb_reg,
norm_eps,
norm_max,
kill_edit,
draw_edit,
swap_memory,
use_beam_decoder=False,
use_dropout=False,
no_insert_delete_attn=False,
enable_vae=True):
batch_size = tf.shape(source_words)[0]
# [batch]
base_len = seq.length_pre_embedding(base_words)
src_len = seq.length_pre_embedding(source_words)
tgt_len = seq.length_pre_embedding(target_words)
iw_len = seq.length_pre_embedding(insert_words)
dw_len = seq.length_pre_embedding(delete_words)
# variable of shape [vocab_size, embed_dim]
embeddings = vocab.get_embeddings()
# [batch x max_len x embed_dim]
base_word_embeds = vocab.embed_tokens(base_words)
# src_word_embeds = vocab.embed_tokens(source_words)
# tgt_word_embeds = vocab.embed_tokens(target_words)
insert_word_embeds = vocab.embed_tokens(insert_words)
delete_word_embeds = vocab.embed_tokens(delete_words)
# [batch x max_len x rnn_out_dim], [batch x rnn_out_dim]
base_sent_hidden_states, base_sent_embed = encoder.source_sent_encoder(
base_word_embeds,
base_len,
hidden_dim,
num_encoder_layers,
use_dropout=use_dropout,
dropout_keep=dropout_keep,
swap_memory=swap_memory)
# [batch x edit_dim]
if kill_edit:
edit_vector = tf.zeros(shape=(batch_size, edit_dim))
else:
if draw_edit:
edit_vector = edit_encoder.random_noise_encoder(
batch_size, edit_dim, norm_max)
else:
edit_vector = edit_encoder.accumulator_encoder(
insert_word_embeds,
delete_word_embeds,
iw_len,
dw_len,
edit_dim,
lamb_reg,
norm_eps,
norm_max,
dropout_keep,
enable_vae=enable_vae)
# [batch x agenda_dim]
input_agenda = agn.linear(base_sent_embed, edit_vector, agenda_dim)
train_dec_inp, train_dec_inp_len, \
train_dec_out, train_dec_out_len = prepare_decoder_input_output(target_words, tgt_len, vocab_table)
train_decoder = decoder.train_decoder(
input_agenda,
embeddings,
train_dec_inp,
base_sent_hidden_states,
insert_word_embeds,
delete_word_embeds,
train_dec_inp_len,
base_len,
iw_len,
dw_len,
attn_dim,
hidden_dim,
num_decoder_layers,
swap_memory,
enable_dropout=use_dropout,
dropout_keep=dropout_keep,
no_insert_delete_attn=no_insert_delete_attn)
if use_beam_decoder:
infr_decoder = decoder.beam_eval_decoder(
input_agenda,
embeddings,
vocab.get_token_id(vocab.START_TOKEN, vocab_table),
vocab.get_token_id(vocab.STOP_TOKEN, vocab_table),
base_sent_hidden_states,
insert_word_embeds,
delete_word_embeds,
base_len,
iw_len,
dw_len,
attn_dim,
hidden_dim,
num_decoder_layers,
max_sent_length,
beam_width,
swap_memory,
enable_dropout=use_dropout,
dropout_keep=dropout_keep,
no_insert_delete_attn=no_insert_delete_attn)
else:
infr_decoder = decoder.greedy_eval_decoder(
input_agenda,
embeddings,
vocab.get_token_id(vocab.START_TOKEN, vocab_table),
vocab.get_token_id(vocab.STOP_TOKEN, vocab_table),
base_sent_hidden_states,
insert_word_embeds,
delete_word_embeds,
base_len,
iw_len,
dw_len,
attn_dim,
hidden_dim,
num_decoder_layers,
max_sent_length,
swap_memory,
enable_dropout=use_dropout,
dropout_keep=dropout_keep,
no_insert_delete_attn=no_insert_delete_attn)
return train_decoder, infr_decoder, train_dec_out, train_dec_out_len
def editor_train(base_words,
extended_base_words,
output_words,
extended_output_words,
source_words,
target_words,
insert_words,
delete_words,
oov,
vocab_size,
hidden_dim,
agenda_dim,
edit_dim,
micro_edit_ev_dim,
num_heads,
num_encoder_layers,
num_decoder_layers,
attn_dim,
beam_width,
ctx_hidden_dim,
ctx_hidden_layer,
wa_hidden_dim,
wa_hidden_layer,
meve_hidden_dim,
meve_hidden_layers,
recons_dense_layers,
max_sent_length,
dropout_keep,
lamb_reg,
norm_eps,
norm_max,
kill_edit,
draw_edit,
swap_memory,
use_beam_decoder=False,
use_dropout=False,
no_insert_delete_attn=False,
enable_vae=True):
batch_size = tf.shape(source_words)[0]
# [batch]
base_len = seq.length_pre_embedding(base_words)
output_len = seq.length_pre_embedding(extended_output_words)
src_len = seq.length_pre_embedding(source_words)
tgt_len = seq.length_pre_embedding(target_words)
iw_len = seq.length_pre_embedding(insert_words)
dw_len = seq.length_pre_embedding(delete_words)
# variable of shape [vocab_size, embed_dim]
embeddings = vocab.get_embeddings()
# [batch x max_len x embed_dim]
base_word_embeds = vocab.embed_tokens(base_words)
output_word_embeds = vocab.embed_tokens(output_words)
# src_word_embeds = vocab.embed_tokens(source_words)
# tgt_word_embeds = vocab.embed_tokens(target_words)
insert_word_embeds = vocab.embed_tokens(insert_words)
delete_word_embeds = vocab.embed_tokens(delete_words)
# [batch x max_len x rnn_out_dim], [batch x rnn_out_dim]
base_sent_hidden_states, base_sent_embed = encoder.source_sent_encoder(
base_word_embeds,
base_len,
hidden_dim,
num_encoder_layers,
use_dropout=use_dropout,
dropout_keep=dropout_keep,
swap_memory=swap_memory)
assert kill_edit == False and draw_edit == False
# [batch x edit_dim]
if kill_edit:
edit_vector = tf.zeros(shape=(batch_size, edit_dim))
else:
if draw_edit:
edit_vector = random_noise_encoder(batch_size, edit_dim, norm_max)
else:
edit_vector = accumulator_encoder(insert_word_embeds,
delete_word_embeds,
iw_len,
dw_len,
edit_dim,
lamb_reg,
norm_eps,
norm_max,
dropout_keep,
enable_vae=enable_vae)
wa_inserted, wa_deleted = (tf.constant([[0]]), tf.constant(
[[0]])), (tf.constant([[0]]), tf.constant([[0]]))
# [batch x agenda_dim]
base_agenda = agn.linear(base_sent_embed, edit_vector, agenda_dim)
train_dec_inp, train_dec_inp_len, \
train_dec_out, train_dec_out_len = prepare_decoder_input_output(output_words, extended_output_words, output_len)
train_dec_inp_extended = prepare_decoder_inputs(extended_output_words,
tf.cast(-1, tf.int64))
train_decoder = decoder.train_decoder(
base_agenda,
embeddings,
extended_base_words,
oov,
train_dec_inp,
train_dec_inp_extended,
base_sent_hidden_states,
wa_inserted,
wa_deleted,
train_dec_inp_len,
base_len,
src_len,
tgt_len,
vocab_size,
attn_dim,
hidden_dim,
num_decoder_layers,
swap_memory,
enable_dropout=use_dropout,
dropout_keep=dropout_keep,
no_insert_delete_attn=no_insert_delete_attn)
if use_beam_decoder:
infr_decoder = decoder.beam_eval_decoder(
base_agenda,
embeddings,
extended_base_words,
oov,
vocab.get_token_id(vocab.START_TOKEN),
vocab.get_token_id(vocab.STOP_TOKEN),
base_sent_hidden_states,
wa_inserted,
wa_deleted,
base_len,
src_len,
tgt_len,
vocab_size,
attn_dim,
hidden_dim,
num_decoder_layers,
max_sent_length,
beam_width,
swap_memory,
enable_dropout=use_dropout,
dropout_keep=dropout_keep,
no_insert_delete_attn=no_insert_delete_attn)
else:
infr_decoder = decoder.greedy_eval_decoder(
base_agenda,
embeddings,
extended_base_words,
oov,
vocab.get_token_id(vocab.START_TOKEN),
vocab.get_token_id(vocab.STOP_TOKEN),
base_sent_hidden_states,
wa_inserted,
wa_deleted,
base_len,
src_len,
tgt_len,
vocab_size,
attn_dim,
hidden_dim,
num_decoder_layers,
max_sent_length,
swap_memory,
enable_dropout=use_dropout,
dropout_keep=dropout_keep,
no_insert_delete_attn=no_insert_delete_attn)
edit_vector_recons = output_words_to_edit_vector(
output_word_embeds, output_len, edit_dim, ctx_hidden_dim,
ctx_hidden_layer, recons_dense_layers, swap_memory)
optimizer.add_reconst_loss(edit_vector, edit_vector_recons)
return train_decoder, infr_decoder, train_dec_out, train_dec_out_len