def compute_ECM_loss(source_ids,
target_ids,
sequence_mask,
choice_qs,
embeddings,
enc_num_layers,
enc_num_units,
enc_cell_type,
enc_bidir,
dec_num_layers,
dec_num_units,
dec_cell_type,
state_pass,
num_emo,
emo_cat,
emo_cat_units,
emo_int_units,
infer_batch_size,
beam_size=None,
max_iter=20,
attn_num_units=128,
l2_regularize=None,
name="ECM"):
"""
Creates an ECM model and returns CE loss plus regularization terms.
choice_qs: [batch_size, max_time], true choice btw generic/emo words
emo_cat: [batch_size], emotion categories of each target sequence
Returns
CE: cross entropy, used to compute perplexity
total_loss: objective of the entire model
train_outs: (cell, log_probs, alphas, final_int_mem_states)
alphas - predicted choices
infer_outputs: namedtuple(logits, ids), [batch_size, max_time, d]
"""
with tf.name_scope(name):
# build encoder
encoder_outputs, encoder_states = build_encoder(embeddings,
source_ids,
enc_num_layers,
enc_num_units,
enc_cell_type,
bidir=enc_bidir,
name="%s_encoder" %
name)
# build decoder: logits, [batch_size, max_time, vocab_size]
cell, train_outputs, infer_outputs = build_ECM_decoder(
encoder_outputs,
encoder_states,
embeddings,
dec_num_layers,
dec_num_units,
dec_cell_type,
num_emo,
emo_cat,
emo_cat_units,
emo_int_units,
state_pass,
infer_batch_size,
attn_num_units,
target_ids,
beam_size,
max_iter,
name="%s_decoder" % name)
g_logits, e_logits, alphas, int_M_emo = train_outputs
g_probs = tf.nn.softmax(g_logits) * (1 - alphas)
e_probs = tf.nn.softmax(e_logits) * alphas
train_log_probs = tf.log(g_probs + e_probs)
with tf.name_scope('loss'):
final_ids = tf.pad(target_ids, [[0, 0], [0, 1]], constant_values=1)
alphas = tf.squeeze(alphas, axis=-1)
choice_qs = tf.pad(choice_qs, [[0, 0], [0, 1]], constant_values=0)
# compute losses
g_losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=g_logits, labels=final_ids) - tf.log(1 - alphas)
e_losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=e_logits, labels=final_ids) - tf.log(alphas)
losses = g_losses * (1 - choice_qs) + e_losses * choice_qs
# alpha and internal memory regularizations
alpha_reg = tf.reduce_mean(choice_qs * -tf.log(alphas))
int_mem_reg = tf.reduce_mean(tf.norm(int_M_emo, axis=1))
losses = tf.boolean_mask(losses[:, :-1], sequence_mask)
reduced_loss = tf.reduce_mean(losses) + alpha_reg + int_mem_reg
# prepare for perplexity computations
CE = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=train_log_probs, labels=final_ids)
CE = tf.boolean_mask(CE[:, :-1], sequence_mask)
CE = tf.reduce_sum(CE)
train_outs = (cell, train_log_probs, alphas, int_M_emo)
if l2_regularize is None:
return CE, reduced_loss, train_outs, infer_outputs
else:
l2_loss = tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables()
if not ('bias' in v.name)
])
total_loss = reduced_loss + l2_regularize * l2_loss
return CE, total_loss, train_outs, infer_outputs
def compute_loss(source_ids,
target_ids,
sequence_mask,
embeddings,
enc_num_layers,
enc_num_units,
enc_cell_type,
enc_bidir,
dec_num_layers,
dec_num_units,
dec_cell_type,
state_pass,
infer_batch_size,
infer_type="greedy",
beam_size=None,
max_iter=20,
attn_wrapper=None,
attn_num_units=128,
l2_regularize=None,
name="Seq2seq"):
"""
Creates a Seq2seq model and returns cross entropy loss.
"""
with tf.name_scope(name):
# build encoder
encoder_outputs, encoder_states = build_encoder(embeddings,
source_ids,
enc_num_layers,
enc_num_units,
enc_cell_type,
bidir=enc_bidir,
name="%s_encoder" %
name)
# build decoder: logits, [batch_size, max_time, vocab_size]
train_logits, infer_outputs = build_decoder(encoder_outputs,
encoder_states,
embeddings,
dec_num_layers,
dec_num_units,
dec_cell_type,
state_pass,
infer_batch_size,
attn_wrapper,
attn_num_units,
target_ids,
infer_type,
beam_size,
max_iter,
name="%s_decoder" % name)
# compute loss
with tf.name_scope('loss'):
final_ids = tf.pad(target_ids, [[0, 0], [0, 1]], constant_values=1)
losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=train_logits, labels=final_ids)
losses = tf.boolean_mask(losses[:, :-1], sequence_mask)
reduced_loss = tf.reduce_mean(losses)
CE = tf.reduce_sum(losses)
if l2_regularize is None:
return CE, reduced_loss, train_logits, infer_outputs
else:
l2_loss = tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables()
if not ('bias' in v.name)
])
total_loss = reduced_loss + l2_regularize * l2_loss
return CE, total_loss, train_logits, infer_outputs
def compute_loss(source_ids,
target_ids,
sequence_mask,
choice_qs,
embeddings,
enc_num_layers,
enc_num_units,
enc_cell_type,
enc_bidir,
dec_num_layers,
dec_num_units,
dec_cell_type,
state_pass,
num_emo,
emo_cat,
emo_cat_units,
emo_int_units,
infer_batch_size,
spectrogram,
word_config,
num_per,
person_ids,
per_cat_units,
spectrogram_config,
loss_weight,
lstm_int_num,
is_train,
is_pretrain,
lexicons_ids,
beam_size=None,
max_iter=20,
attn_num_units=128,
l2_regularize=None,
name="PEC",
is_0le=False):
"""
Creates an ECM model and returns CE loss plus regularization terms.
choice_qs: [batch_size, max_time], true choice btw generic/emo words
emo_cat: [batch_size], emotion categories of each target sequence
Returns
CE: cross entropy, used to compute perplexity
total_loss: objective of the entire model
train_outs: (cell, log_probs, alphas, final_int_mem_states)
alphas - predicted choices
infer_outputs: namedtuple(logits, ids), [batch_size, max_time, d]
"""
with tf.name_scope(name):
# build encoder
encoder_outputs, encoder_states = build_encoder(embeddings,
source_ids,
enc_num_layers,
enc_num_units,
enc_cell_type,
bidir=enc_bidir,
name="%s_encoder" %
name)
# gain lexicons embeddings
if is_pretrain or is_0le:
lexicons_embeddings = tf.zeros(
[embeddings.shape[0].value, embeddings.shape[1].value])
lexicons_embeddings = tf.nn.embedding_lookup(
lexicons_embeddings, lexicons_ids)
else:
lexicons_embeddings = tf.nn.embedding_lookup(
embeddings, lexicons_ids)
lex_emb = tf.nn.embedding_lookup(lexicons_embeddings, person_ids)
# extract multi-modal feature
word_embeddings = tf.nn.embedding_lookup(embeddings, source_ids)
p_model = PWrapper(word_embeddings, spectrogram, person_ids,
word_config, spectrogram_config, lstm_int_num,
is_train)
multimodal_outputs = p_model.build_p()
# build decoder: logits, [batch_size, max_time, vocab_size]
cell, train_outputs, infer_outputs = build_PEC_decoder(
encoder_outputs,
encoder_states,
multimodal_outputs,
lex_emb,
embeddings,
num_per,
person_ids,
per_cat_units,
dec_num_layers,
dec_num_units,
dec_cell_type,
num_emo,
emo_cat,
emo_cat_units,
emo_int_units,
state_pass,
infer_batch_size,
attn_num_units,
target_ids,
beam_size,
max_iter,
is_pretrain,
name="%s_decoder" % name)
g_logits, e_logits, alphas, int_M_emo = train_outputs
g_probs = tf.nn.softmax(g_logits) * (1 - alphas)
e_probs = tf.nn.softmax(e_logits) * alphas
train_log_probs = tf.log(g_probs + e_probs)
with tf.name_scope('loss'):
final_ids = tf.pad(target_ids, [[0, 0], [0, 1]], constant_values=1)
alphas = tf.squeeze(alphas, axis=-1)
choice_qs = tf.pad(choice_qs, [[0, 0], [0, 1]], constant_values=0)
CE = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=train_log_probs, labels=final_ids)
# compute losses
g_losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=g_logits, labels=final_ids) - tf.log(1 - alphas)
e_losses = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=e_logits, labels=final_ids) - tf.log(alphas)
losses = g_losses * (1 - choice_qs) + e_losses * choice_qs
# prepare for perplexity computations and person classifier
target_embeddings = tf.nn.embedding_lookup(embeddings, final_ids)
word_pros = -1 * tf.log(
tf.reduce_sum(
tf.one_hot(final_ids, int(train_log_probs.shape[-1])) *
tf.nn.softmax(train_log_probs, axis=-1),
axis=-1)[:, :-1])
# word_pros = CE[:, :-1]
Ewe = tf.reshape(word_pros, [-1, int(word_pros.shape[-1]), 1]) \
* tf.reshape(tf.cast(sequence_mask, tf.float32), [-1, int(sequence_mask.shape[-1]), 1]) \
* target_embeddings[:, :-1]
classsifier = tf.layers.Dense(num_per, activation=None)
Ewe = classsifier(tf.reduce_mean(Ewe, axis=1))
# tmp_embeddings = tf.concat([embeddings, embeddings], 0)
# infer_log_pro = infer_outputs.logits
# infer_onehot = tf.one_hot(infer_outputs.ids, infer_outputs.logits.shape[-1].value)
# infer_log_pro = -1 * tf.reduce_sum(infer_log_pro * infer_onehot, axis=-1)[:, :, 0]
# infer_log_pro = tf.expand_dims(infer_log_pro, axis=-1)
# infer_emb = tf.nn.embedding_lookup(tmp_embeddings, infer_outputs.ids[:, :, 0])
# infer_emb = infer_log_pro * infer_emb
# person_probs = tf.nn.softmax(classsifier(tf.reduce_mean(infer_emb, axis=1)), axis=-1)
# person_onehot = tf.one_hot(person_ids, num_per)
# score = tf.reduce_sum(person_probs * person_onehot, axis=-1)
infer_ids = infer_outputs.ids[:, :, 0] % int(embeddings.shape[0])
infer_emb = tf.nn.embedding_lookup(embeddings, infer_ids)
infer_log_pro = tf.exp(infer_outputs.logits[:, :, 0])
infer_gen_pro, infer_emo_pro = tf.split(infer_log_pro,
[42003, 42003], -1)
infer_pro = tf.log(infer_gen_pro + infer_emo_pro)
infer_word_pros = -1 * tf.log(
tf.reduce_sum(tf.one_hot(infer_ids, int(infer_pro.shape[-1])) *
tf.nn.softmax(infer_pro, axis=-1),
axis=-1))
infer_sequence_mask = tf.not_equal(infer_ids, 1)
infer_Ewe = tf.expand_dims(infer_word_pros, -1)\
* tf.expand_dims(tf.cast(infer_sequence_mask, tf.float32), -1) \
* infer_emb
infer_Ewe = classsifier(tf.reduce_sum(infer_Ewe, axis=1) / 40)
score = tf.exp(-1 * tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=infer_Ewe, labels=person_ids))
# alpha and internal memory regularizations
person_loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=Ewe, labels=person_ids))
alpha_reg = tf.reduce_mean(choice_qs * -tf.log(alphas))
int_mem_reg = tf.reduce_mean(tf.norm(int_M_emo, axis=1))
losses = tf.boolean_mask(losses[:, :-1], sequence_mask)
if is_pretrain:
reduced_loss = tf.reduce_mean(losses) + alpha_reg + int_mem_reg
else:
reduced_loss = tf.reduce_mean(
losses
) + alpha_reg + int_mem_reg + loss_weight * person_loss
train_outs = (cell, train_log_probs, alphas, int_M_emo)
CE = tf.boolean_mask(CE[:, :-1], sequence_mask)
CE = tf.reduce_sum(CE)
if l2_regularize is None:
return CE, reduced_loss, loss_weight * person_loss, train_outs, infer_outputs, score
else:
l2_loss = tf.add_n([
tf.nn.l2_loss(v) for v in tf.trainable_variables()
if not ('bias' in v.name)
])
total_loss = reduced_loss + l2_regularize * l2_loss
return CE, total_loss, loss_weight * person_loss, train_outs, infer_outputs, score