def build_model(self):
# input layer (batch_size, n_steps, input_dim)
self.ques_vecs = tf.placeholder(
tf.float32, [None, self.max_words, self.input_ques_dim])
self.ques_len = tf.placeholder(tf.int32, [None])
self.frame_vecs = tf.placeholder(
tf.float32, [None, self.max_frames, self.input_video_dim])
self.frame_len = tf.placeholder(tf.int32, [None])
self.batch_size = tf.placeholder(tf.int32, [])
self.is_training = tf.placeholder(tf.bool)
self.gt_predict = tf.placeholder(tf.float32, [None, self.max_frames])
self.gt_windows = tf.placeholder(tf.float32, [None, 2])
self.frame_mask = tf.sequence_mask(self.frame_len,
maxlen=self.max_frames)
self.ques_mask = tf.sequence_mask(self.ques_len, maxlen=self.max_words)
with tf.variable_scope("Frame_Embedding_Encoder_Layer"):
input_frame_vecs = tf.contrib.layers.dropout(
self.frame_vecs, self.dropout, is_training=self.is_training)
frame_embedding, _ = layers.dynamic_origin_bilstm_layer(
input_frame_vecs,
self.hidden_size,
'frame_embedding',
input_len=self.frame_len)
frame_embedding = tf.contrib.layers.dropout(
frame_embedding, self.dropout, is_training=self.is_training)
with tf.variable_scope("Ques_Embedding_Encoder_Layer"):
input_ques_vecs = tf.contrib.layers.dropout(
self.ques_vecs, self.dropout, is_training=self.is_training)
ques_embedding, ques_states = layers.dynamic_origin_bilstm_layer(
input_ques_vecs,
self.hidden_size,
'ques_embedding',
input_len=self.ques_len)
ques_embedding = tf.contrib.layers.dropout(
ques_embedding, self.dropout, is_training=self.is_training)
q_feature = tf.concat([ques_states[0][1], ques_states[1][1]], 1)
self.q_feature = tf.contrib.layers.dropout(
q_feature, self.dropout, is_training=self.is_training)
with tf.variable_scope("Context_to_Query_Attention_Layer"):
# att_score = tf.matmul(frame_embedding, ques_embedding, transpose_b=True) # M*N1*K ** M*N2*K --> M*N1*N2
# att_score = tf.nn.softmax(mask_logits(att_score, mask=tf.expand_dims(self.ques_mask, 1)))
#
# length = tf.cast(tf.shape(ques_embedding), tf.float32)
# att_out = tf.matmul(att_score, ques_embedding) * length[1] * tf.sqrt(
# 1.0 / length[1]) # M*N1*N2 ** M*N2*K --> M*N1*k
#
# attention_outputs = tf.concat([frame_embedding, att_out, tf.multiply(frame_embedding,att_out)])
att_score = tf.matmul(
frame_embedding, ques_embedding,
transpose_b=True) # M*N1*K ** M*N2*K --> M*N1*N2
mask_q = tf.expand_dims(self.ques_mask, 1)
S_ = tf.nn.softmax(layers.mask_logits(att_score, mask=mask_q))
mask_v = tf.expand_dims(self.frame_mask, 2)
S_T = tf.transpose(
tf.nn.softmax(layers.mask_logits(att_score, mask=mask_v),
axis=1), (0, 2, 1))
self.v2q = tf.matmul(S_, ques_embedding)
self.q2v = tf.matmul(tf.matmul(S_, S_T), frame_embedding)
attention_outputs = tf.concat([
frame_embedding, self.v2q, frame_embedding * self.v2q,
frame_embedding * self.q2v
], 2)
with tf.variable_scope("Model_Encoder_Layer"):
attention_outputs = tf.contrib.layers.dropout(
attention_outputs, self.dropout, is_training=self.is_training)
model_outputs, _ = layers.dynamic_origin_bilstm_layer(
attention_outputs,
self.hidden_size,
'model_layer',
input_len=self.frame_len)
model_outputs = tf.contrib.layers.dropout(
model_outputs, self.dropout, is_training=self.is_training)
with tf.variable_scope("Output_Layer"):
logit_score = layers.correlation_layer(model_outputs,
self.q_feature,
self.hidden_size,
scope_name='output_layer')
# logit_score = layers.linear_layer_3d(model_outputs, 1, scope_name='output_layer')
# logit_score = tf.squeeze(logit_score, 2)
logit_loss = tf.nn.sigmoid_cross_entropy_with_logits(
logits=logit_score, labels=self.gt_predict)
avg_logit_loss = tf.reduce_mean(tf.reduce_sum(logit_loss, 1))
self.G_variables = tf.trainable_variables()
G_regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in self.G_variables])
G_reg_loss = self.regularization_beta * G_regularization_cost
ground_prod = tf.nn.softmax(
layers.mask_logits(self.gt_predict, self.gt_predict))
ground_v_feature = tf.reduce_sum(
tf.multiply(model_outputs, tf.expand_dims(ground_prod, 2)), 1)
ground_out = self.discriminator(ground_v_feature, self.q_feature)
generated_prod = tf.nn.sigmoid(logit_score) / tf.reduce_sum(
tf.nn.sigmoid(logit_score), keepdims=True, axis=1)
generated_v_feature = tf.reduce_sum(
tf.multiply(model_outputs, tf.expand_dims(generated_prod, 2)),
1)
generated_out = self.discriminator(generated_v_feature,
self.q_feature,
reuse_flag=True)
all_variable = tf.trainable_variables()
self.D_variables = [
vv for vv in all_variable if vv not in self.G_variables
]
D_regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in self.D_variables])
D_reg_loss = self.regularization_beta * D_regularization_cost
ground_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(ground_out), logits=ground_out))
generated_loss = tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.zeros_like(generated_out), logits=generated_out))
regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in all_variable])
self.reg_loss = self.regularization_beta * regularization_cost
self.dist_loss = avg_logit_loss
self.G_pre_loss = avg_logit_loss + G_reg_loss
self.D_loss = ground_loss + generated_loss + D_reg_loss
scale = self.max_frames / 5
self.G_loss = avg_logit_loss + G_reg_loss + scale * tf.reduce_mean(
tf.nn.sigmoid_cross_entropy_with_logits(
labels=tf.ones_like(generated_out), logits=generated_out))
self.frame_score = tf.nn.sigmoid(logit_score)
with tf.variable_scope('Pointer_Layer'):
score_dist = tf.nn.sigmoid(logit_score)
score_dist = conv_utils.normalize(score_dist, scope='layer_normal')
output = tf.nn.relu(
conv_utils.conv1d_with_bias(tf.expand_dims(score_dist, 2), 1,
16, 5))
# output = tf.contrib.layers.dropout(output, self.dropout, is_training=self.is_training)
output = tf.nn.relu(conv_utils.conv1d_with_bias(output, 2, 32, 10))
# output = tf.contrib.layers.dropout(output, self.dropout, is_training=self.is_training)
output = tf.nn.relu(conv_utils.conv1d_with_bias(output, 3, 64, 20))
# output = tf.contrib.layers.dropout(output, self.dropout, is_training=self.is_training)
output = tf.nn.relu(conv_utils.conv1d_with_bias(output, 4, 1, 10))
# output = tf.contrib.layers.dropout(output, self.dropout, is_training=self.is_training)
# output = tf.nn.relu(layers.linear_layer(output, self.hidden_size,scope_name='pointer_1'))
# output = tf.nn.relu(layers.linear_layer(output, self.hidden_size/2,scope_name='pointer_2'))
# output = tf.nn.relu(layers.linear_layer(output, self.hidden_size/4,scope_name='pointer_3'))
output = layers.linear_layer(tf.squeeze(output, 2),
2,
scope_name='pointrt_output')
self.predict_start_end = output
gt_start_end = self.gt_windows
pointer_loss = tf.reduce_mean(
tf.square(tf.subtract(self.predict_start_end, gt_start_end)))
all_variable = tf.trainable_variables()
self.pn_variables = [
vv for vv in all_variable
if vv not in self.G_variables and vv not in self.D_variables
]
pn_regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in self.pn_variables])
self.pn_loss = pointer_loss + self.regularization_beta * pn_regularization_cost
print(self.G_variables)
print(self.D_variables)
print(self.pn_variables)
def build_model(self):
# input layer (batch_size, n_steps, input_dim)
self.ques_vecs = tf.placeholder(
tf.float32, [None, self.max_words, self.input_ques_dim])
self.ques_len = tf.placeholder(tf.int32, [None])
self.frame_vecs = tf.placeholder(
tf.float32, [None, self.max_frames, self.input_video_dim])
self.frame_len = tf.placeholder(tf.int32, [None])
self.batch_size = tf.placeholder(tf.int32, [])
self.is_training = tf.placeholder(tf.bool)
self.gt_predict = tf.placeholder(tf.float32, [None, self.max_frames])
self.gt_windows = tf.placeholder(tf.float32, [None, 2])
self.frame_mask = tf.sequence_mask(self.frame_len,
maxlen=self.max_frames)
self.ques_mask = tf.sequence_mask(self.ques_len, maxlen=self.max_words)
with tf.variable_scope("Frame_Embedding_Encoder_Layer"):
frame_next_layer = tf.contrib.layers.dropout(
self.frame_vecs, self.dropout, is_training=self.is_training)
frame_next_layer = conv_utils.linear_mapping(
frame_next_layer,
self.hidden_size,
dropout=self.dropout,
var_scope_name="linear_mapping_before_cnn")
frame_next_layer = transformer.normalize(frame_next_layer)
frame_next_layer += transformer.positional_encoding_v2(
frame_next_layer,
num_units=self.hidden_size,
zero_pad=False,
scale=False,
scope="enc_pe")
for i in range(3):
with tf.variable_scope("stack_%s" % i):
frame_next_layer = conv_utils.conv_encoder_stack(
frame_next_layer,
[self.hidden_size, self.hidden_size, self.hidden_size],
[3, 3, 3], {
'src': self.dropout,
'hid': self.dropout
},
mode=self.is_training)
frame_next_layer = transformer.multihead_attention(
queries=frame_next_layer,
keys=frame_next_layer,
num_units=self.hidden_size,
num_heads=4,
dropout_rate=1 - self.dropout,
is_training=self.is_training,
causality=False)
frame_next_layer = transformer.feedforward(
frame_next_layer,
num_units=[2 * self.hidden_size, self.hidden_size],
is_training=self.is_training)
frame_embedding = tf.contrib.layers.dropout(
frame_next_layer, self.dropout, is_training=self.is_training)
with tf.variable_scope("Ques_Embedding_Encoder_Layer"):
ques_next_layer = tf.contrib.layers.dropout(
self.ques_vecs, self.dropout, is_training=self.is_training)
ques_next_layer = conv_utils.linear_mapping(
ques_next_layer,
self.hidden_size,
dropout=self.dropout,
var_scope_name="linear_mapping_before_cnn")
ques_next_layer = transformer.normalize(ques_next_layer)
ques_next_layer += transformer.positional_encoding_v2(
ques_next_layer,
num_units=self.hidden_size,
zero_pad=False,
scale=False,
scope="enc_pe")
for i in range(1):
with tf.variable_scope("stack_%s" % i):
ques_next_layer = conv_utils.conv_encoder_stack(
ques_next_layer, [self.hidden_size, self.hidden_size],
[3, 3], {
'src': self.dropout,
'hid': self.dropout
},
mode=self.is_training)
ques_next_layer = transformer.multihead_attention(
queries=ques_next_layer,
keys=ques_next_layer,
num_units=self.hidden_size,
num_heads=4,
dropout_rate=1 - self.dropout,
is_training=self.is_training,
causality=False)
ques_next_layer = transformer.feedforward(
ques_next_layer,
num_units=[2 * self.hidden_size, self.hidden_size],
is_training=self.is_training)
ques_embedding = tf.contrib.layers.dropout(
ques_next_layer, self.dropout, is_training=self.is_training)
# q_feature, _ = layers.weight_attention_layer(ques_embedding,self.hidden_size,scope_name='q_feature')
ques_mask_embedding = layers.mask_zero(
ques_next_layer, tf.expand_dims(self.ques_mask, 2))
q_feature = tf.reduce_sum(
ques_mask_embedding, axis=1) / tf.expand_dims(
tf.cast(self.ques_len, tf.float32), 1)
# q_feature = tf.reduce_mean(ques_next_layer,axis=1)
print(q_feature.shape)
self.q_feature = tf.contrib.layers.dropout(
q_feature, self.dropout, is_training=self.is_training)
with tf.variable_scope("Context_to_Query_Attention_Layer"):
att_score = tf.matmul(
frame_embedding, ques_embedding,
transpose_b=True) # M*N1*K ** M*N2*K --> M*N1*N2
mask_q = tf.expand_dims(self.ques_mask, 1)
S_ = tf.nn.softmax(layers.mask_logits(att_score, mask=mask_q))
mask_v = tf.expand_dims(self.frame_mask, 2)
S_T = tf.transpose(
tf.nn.softmax(layers.mask_logits(att_score, mask=mask_v),
axis=1), (0, 2, 1))
self.v2q = tf.matmul(S_, ques_embedding)
self.q2v = tf.matmul(tf.matmul(S_, S_T), frame_embedding)
attention_outputs = tf.concat([
frame_embedding, self.v2q, frame_embedding * self.v2q,
frame_embedding * self.q2v
], 2)
with tf.variable_scope("Model_Encoder_Layer"):
model_next_layer = conv_utils.linear_mapping(
attention_outputs,
self.hidden_size,
dropout=self.dropout,
var_scope_name="linear_mapping_before_model_layer")
model_next_layer = transformer.normalize(model_next_layer)
for i in range(2):
with tf.variable_scope("stack_%s" % i):
model_next_layer = conv_utils.conv_encoder_stack(
model_next_layer, [self.hidden_size, self.hidden_size],
[3, 3], {
'src': self.dropout,
'hid': self.dropout
},
mode=self.is_training)
model_next_layer = transformer.multihead_attention(
queries=model_next_layer,
keys=model_next_layer,
num_units=self.hidden_size,
num_heads=4,
dropout_rate=1 - self.dropout,
is_training=self.is_training,
causality=False)
model_next_layer = transformer.feedforward(
model_next_layer,
num_units=[2 * self.hidden_size, self.hidden_size],
is_training=self.is_training)
model_outputs = model_next_layer
with tf.variable_scope("Output_Layer"):
# logit_score = layers.correlation_layer(model_outputs,self.q_feature,self.hidden_size,scope_name='output_layer')
logit_score = layers.linear_layer_3d(model_outputs,
1,
scope_name='output_layer')
logit_score = tf.squeeze(logit_score, 2)
logit_loss = tf.nn.sigmoid_cross_entropy_with_logits(
logits=logit_score, labels=self.gt_predict)
avg_logit_loss = tf.reduce_mean(tf.reduce_sum(logit_loss, 1))
self.G_variables = tf.trainable_variables()
G_regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in self.G_variables])
self.test_loss = avg_logit_loss
self.loss = avg_logit_loss + self.regularization_beta * G_regularization_cost
self.frame_score = tf.nn.sigmoid(logit_score)
with tf.variable_scope('Pointer_Layer'):
score_dist = tf.nn.sigmoid(logit_score)
score_dist = conv_utils.normalize(score_dist, scope='layer_normal')
output = tf.nn.relu(
conv_utils.conv1d_with_bias(tf.expand_dims(score_dist, 2), 1,
16, 5))
output = tf.nn.relu(conv_utils.conv1d_with_bias(output, 2, 32, 10))
output = tf.nn.relu(conv_utils.conv1d_with_bias(output, 3, 64, 20))
output = tf.nn.relu(conv_utils.conv1d_with_bias(output, 4, 1, 10))
output = layers.linear_layer(tf.squeeze(output, 2),
2,
scope_name='pointrt_output')
self.predict_start_end = output
gt_start_end = self.gt_windows
pointer_loss = tf.reduce_mean(
tf.square(tf.subtract(self.predict_start_end, gt_start_end)))
all_variable = tf.trainable_variables()
self.pn_variables = [
vv for vv in all_variable if vv not in self.G_variables
]
pn_regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in self.pn_variables])
self.pn_loss = pointer_loss + self.regularization_beta * pn_regularization_cost
def build_model(self):
# input layer (batch_size, n_steps, input_dim)
self.ques_vecs = tf.placeholder(
tf.float32, [None, self.max_words, self.input_ques_dim])
self.ques_len = tf.placeholder(tf.int32, [None])
self.frame_vecs = tf.placeholder(
tf.float32, [None, self.max_frames, self.input_video_dim])
self.frame_len = tf.placeholder(tf.int32, [None])
self.batch_size = tf.placeholder(tf.int32, [])
self.is_training = tf.placeholder(tf.bool)
self.gt_predict = tf.placeholder(tf.float32, [None, self.max_frames])
self.frame_mask = tf.sequence_mask(self.frame_len,
maxlen=self.max_frames)
self.ques_mask = tf.sequence_mask(self.ques_len, maxlen=self.max_words)
with tf.variable_scope("Frame_Embedding_Encoder_Layer"):
frame_next_layer = tf.contrib.layers.dropout(
self.frame_vecs, self.dropout, is_training=self.is_training)
frame_next_layer = conv_utils.linear_mapping(
frame_next_layer,
self.hidden_size,
dropout=self.dropout,
var_scope_name="linear_mapping_before_cnn")
frame_next_layer = transformer.normalize(frame_next_layer)
frame_next_layer += transformer.positional_encoding_v2(
frame_next_layer,
num_units=self.hidden_size,
zero_pad=False,
scale=False,
scope="enc_pe")
for i in range(3):
with tf.variable_scope("stack_%s" % i):
frame_next_layer = conv_utils.conv_encoder_stack(
frame_next_layer,
[self.hidden_size, self.hidden_size, self.hidden_size],
[3, 3, 3], {
'src': self.dropout,
'hid': self.dropout
},
mode=self.is_training)
frame_next_layer = transformer.multihead_attention(
queries=frame_next_layer,
keys=frame_next_layer,
num_units=self.hidden_size,
num_heads=4,
dropout_rate=1 - self.dropout,
is_training=self.is_training,
causality=False)
frame_next_layer = transformer.feedforward(
frame_next_layer,
num_units=[2 * self.hidden_size, self.hidden_size],
is_training=self.is_training)
frame_embedding = tf.contrib.layers.dropout(
frame_next_layer, self.dropout, is_training=self.is_training)
with tf.variable_scope("Ques_Embedding_Encoder_Layer"):
ques_next_layer = tf.contrib.layers.dropout(
self.ques_vecs, self.dropout, is_training=self.is_training)
ques_next_layer = conv_utils.linear_mapping(
ques_next_layer,
self.hidden_size,
dropout=self.dropout,
var_scope_name="linear_mapping_before_cnn")
ques_next_layer = transformer.normalize(ques_next_layer)
ques_next_layer += transformer.positional_encoding_v2(
ques_next_layer,
num_units=self.hidden_size,
zero_pad=False,
scale=False,
scope="enc_pe")
for i in range(1):
with tf.variable_scope("stack_%s" % i):
ques_next_layer = conv_utils.conv_encoder_stack(
ques_next_layer, [self.hidden_size, self.hidden_size],
[3, 3], {
'src': self.dropout,
'hid': self.dropout
},
mode=self.is_training)
ques_next_layer = transformer.multihead_attention(
queries=ques_next_layer,
keys=ques_next_layer,
num_units=self.hidden_size,
num_heads=4,
dropout_rate=1 - self.dropout,
is_training=self.is_training,
causality=False)
ques_next_layer = transformer.feedforward(
ques_next_layer,
num_units=[2 * self.hidden_size, self.hidden_size],
is_training=self.is_training)
ques_embedding = tf.contrib.layers.dropout(
ques_next_layer, self.dropout, is_training=self.is_training)
with tf.variable_scope("Context_to_Query_Attention_Layer"):
att_score = tf.matmul(
frame_embedding, ques_embedding,
transpose_b=True) # M*N1*K ** M*N2*K --> M*N1*N2
mask_q = tf.expand_dims(self.ques_mask, 1)
S_ = tf.nn.softmax(layers.mask_logits(att_score, mask=mask_q))
mask_v = tf.expand_dims(self.frame_mask, 2)
S_T = tf.transpose(
tf.nn.softmax(layers.mask_logits(att_score, mask=mask_v),
axis=1), (0, 2, 1))
self.v2q = tf.matmul(S_, ques_embedding)
self.q2v = tf.matmul(tf.matmul(S_, S_T), frame_embedding)
attention_outputs = tf.concat([
frame_embedding, self.v2q, frame_embedding * self.v2q,
frame_embedding * self.q2v
], 2)
with tf.variable_scope("Model_Encoder_Layer"):
model_next_layer = conv_utils.linear_mapping(
attention_outputs,
self.hidden_size,
dropout=self.dropout,
var_scope_name="linear_mapping_before_model_layer")
model_next_layer = transformer.normalize(model_next_layer)
for i in range(2):
with tf.variable_scope("stack_%s" % i):
model_next_layer = conv_utils.conv_encoder_stack(
model_next_layer, [self.hidden_size, self.hidden_size],
[3, 3], {
'src': self.dropout,
'hid': self.dropout
},
mode=self.is_training)
model_next_layer = transformer.multihead_attention(
queries=model_next_layer,
keys=model_next_layer,
num_units=self.hidden_size,
num_heads=4,
dropout_rate=1 - self.dropout,
is_training=self.is_training,
causality=False)
model_next_layer = transformer.feedforward(
model_next_layer,
num_units=[2 * self.hidden_size, self.hidden_size],
is_training=self.is_training)
model_outputs = model_next_layer
with tf.variable_scope("Output_Layer"):
logit_score = layers.linear_layer_3d(model_outputs,
1,
scope_name='output_layer')
logit_score = tf.squeeze(logit_score, 2)
logit_loss = tf.nn.sigmoid_cross_entropy_with_logits(
logits=logit_score, labels=self.gt_predict)
avg_logit_loss = tf.reduce_mean(tf.reduce_sum(logit_loss, 1))
variables = tf.trainable_variables()
print(len(variables))
regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in variables])
self.test_loss = avg_logit_loss
self.loss = avg_logit_loss + self.regularization_beta * regularization_cost
self.frame_score = tf.nn.sigmoid(logit_score)
def build_model(self):
# input layer (batch_size, n_steps, input_dim)
self.ques_vecs = tf.placeholder(
tf.float32, [None, self.max_words, self.input_ques_dim])
self.ques_len = tf.placeholder(tf.int32, [None])
self.frame_vecs = tf.placeholder(
tf.float32, [None, self.max_frames, self.input_video_dim])
self.frame_len = tf.placeholder(tf.int32, [None])
self.batch_size = tf.placeholder(tf.int32, [])
self.is_training = tf.placeholder(tf.bool)
self.gt_predict = tf.placeholder(tf.float32, [None, self.max_frames])
self.frame_mask = tf.sequence_mask(self.frame_len,
maxlen=self.max_frames)
self.ques_mask = tf.sequence_mask(self.ques_len, maxlen=self.max_words)
with tf.variable_scope("Frame_Embedding_Encoder_Layer"):
input_frame_vecs = tf.contrib.layers.dropout(
self.frame_vecs, self.dropout, is_training=self.is_training)
frame_embedding, _ = layers.dynamic_origin_bilstm_layer(
input_frame_vecs,
self.hidden_size,
'frame_embedding',
input_len=self.frame_len)
frame_embedding = tf.contrib.layers.dropout(
frame_embedding, self.dropout, is_training=self.is_training)
with tf.variable_scope("Ques_Embedding_Encoder_Layer"):
input_ques_vecs = tf.contrib.layers.dropout(
self.ques_vecs, self.dropout, is_training=self.is_training)
ques_embedding, ques_states = layers.dynamic_origin_bilstm_layer(
input_ques_vecs,
self.hidden_size,
'ques_embedding',
input_len=self.ques_len)
ques_embedding = tf.contrib.layers.dropout(
ques_embedding, self.dropout, is_training=self.is_training)
with tf.variable_scope("Context_to_Query_Attention_Layer"):
# att_score = tf.matmul(frame_embedding, ques_embedding, transpose_b=True) # M*N1*K ** M*N2*K --> M*N1*N2
# att_score = tf.nn.softmax(mask_logits(att_score, mask=tf.expand_dims(self.ques_mask, 1)))
#
# length = tf.cast(tf.shape(ques_embedding), tf.float32)
# att_out = tf.matmul(att_score, ques_embedding) * length[1] * tf.sqrt(
# 1.0 / length[1]) # M*N1*N2 ** M*N2*K --> M*N1*k
#
# attention_outputs = tf.concat([frame_embedding, att_out, tf.multiply(frame_embedding,att_out)])
att_score = tf.matmul(
frame_embedding, ques_embedding,
transpose_b=True) # M*N1*K ** M*N2*K --> M*N1*N2
mask_q = tf.expand_dims(self.ques_mask, 1)
S_ = tf.nn.softmax(layers.mask_logits(att_score, mask=mask_q))
mask_v = tf.expand_dims(self.frame_mask, 2)
S_T = tf.transpose(
tf.nn.softmax(layers.mask_logits(att_score, mask=mask_v),
axis=1), (0, 2, 1))
self.v2q = tf.matmul(S_, ques_embedding)
self.q2v = tf.matmul(tf.matmul(S_, S_T), frame_embedding)
attention_outputs = tf.concat([
frame_embedding, self.v2q, frame_embedding * self.v2q,
frame_embedding * self.q2v
], 2)
with tf.variable_scope("Model_Encoder_Layer"):
attention_outputs = tf.contrib.layers.dropout(
attention_outputs, self.dropout, is_training=self.is_training)
model_outputs, _ = layers.dynamic_origin_bilstm_layer(
attention_outputs,
self.hidden_size,
'model_layer',
input_len=self.frame_len)
model_outputs = tf.contrib.layers.dropout(
model_outputs, self.dropout, is_training=self.is_training)
with tf.variable_scope("Output_Layer"):
logit_score = layers.linear_layer_3d(model_outputs,
1,
scope_name='output_layer')
logit_score = tf.squeeze(logit_score, 2)
logit_loss = tf.nn.sigmoid_cross_entropy_with_logits(
logits=logit_score, labels=self.gt_predict)
avg_logit_loss = tf.reduce_mean(tf.reduce_sum(logit_loss, 1))
variables = tf.trainable_variables()
regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in variables])
self.test_loss = avg_logit_loss
self.loss = avg_logit_loss + self.regularization_beta * regularization_cost
self.frame_score = tf.nn.sigmoid(logit_score)
def build_model(self):
# input layer (batch_size, n_steps, input_dim)
self.ques_vecs = tf.placeholder(
tf.float32, [None, self.max_words, self.input_ques_dim])
self.ques_len = tf.placeholder(tf.int32, [None])
self.frame_vecs = tf.placeholder(
tf.float32, [None, self.max_frames, self.input_video_dim])
self.frame_len = tf.placeholder(tf.int32, [None])
self.batch_size = tf.placeholder(tf.int32, [])
self.is_training = tf.placeholder(tf.bool)
self.gt_predict = tf.placeholder(tf.float32, [None, self.max_frames])
self.gt_windows = tf.placeholder(tf.float32, [None, 2])
self.gt_start = tf.placeholder(tf.float32, [None, self.max_frames])
self.gt_end = tf.placeholder(tf.float32, [None, self.max_frames])
self.frame_mask = tf.sequence_mask(self.frame_len,
maxlen=self.max_frames)
self.ques_mask = tf.sequence_mask(self.ques_len, maxlen=self.max_words)
with tf.variable_scope("Frame_Embedding_Encoder_Layer"):
input_frame_vecs = tf.contrib.layers.dropout(
self.frame_vecs, self.dropout, is_training=self.is_training)
# frame_embedding = conv_utils.linear_mapping(input_frame_vecs, self.hidden_size,
# dropout=self.dropout,
# var_scope_name="linear_mapping_before_rnn")
# frame_embedding = transformer.normalize(frame_embedding)
frame_embedding, _ = layers.dynamic_origin_bilstm_layer(
input_frame_vecs,
self.hidden_size,
'frame_embedding',
input_len=self.frame_len)
frame_embedding = tf.contrib.layers.dropout(
frame_embedding, self.dropout, is_training=self.is_training)
with tf.variable_scope("Ques_Embedding_Encoder_Layer"):
input_ques_vecs = tf.contrib.layers.dropout(
self.ques_vecs, self.dropout, is_training=self.is_training)
# ques_embedding = conv_utils.linear_mapping(input_ques_vecs, self.hidden_size,
# dropout=self.dropout,
# var_scope_name="linear_mapping_before_rnn")
# ques_embedding = transformer.normalize(ques_embedding)
ques_embedding, ques_states = layers.dynamic_origin_bilstm_layer(
input_ques_vecs,
self.hidden_size,
'ques_embedding',
input_len=self.ques_len)
ques_embedding = tf.contrib.layers.dropout(
ques_embedding, self.dropout, is_training=self.is_training)
q_feature = tf.concat([ques_states[0][1], ques_states[1][1]], 1)
self.q_feature = tf.contrib.layers.dropout(
q_feature, self.dropout, is_training=self.is_training)
with tf.variable_scope("Context_to_Query_Attention_Layer"):
# att_score = tf.matmul(frame_embedding, ques_embedding, transpose_b=True) # M*N1*K ** M*N2*K --> M*N1*N2
# att_score = tf.nn.softmax(mask_logits(att_score, mask=tf.expand_dims(self.ques_mask, 1)))
#
# length = tf.cast(tf.shape(ques_embedding), tf.float32)
# att_out = tf.matmul(att_score, ques_embedding) * length[1] * tf.sqrt(
# 1.0 / length[1]) # M*N1*N2 ** M*N2*K --> M*N1*k
#
# attention_outputs = tf.concat([frame_embedding, att_out, tf.multiply(frame_embedding,att_out)])
att_score = tf.matmul(
frame_embedding, ques_embedding,
transpose_b=True) # M*N1*K ** M*N2*K --> M*N1*N2
mask_q = tf.expand_dims(self.ques_mask, 1)
S_ = tf.nn.softmax(layers.mask_logits(att_score, mask=mask_q))
mask_v = tf.expand_dims(self.frame_mask, 2)
S_T = tf.transpose(
tf.nn.softmax(layers.mask_logits(att_score, mask=mask_v),
axis=1), (0, 2, 1))
self.v2q = tf.matmul(S_, ques_embedding)
self.q2v = tf.matmul(tf.matmul(S_, S_T), frame_embedding)
attention_outputs = tf.concat([
frame_embedding, self.v2q, frame_embedding * self.v2q,
frame_embedding * self.q2v
], 2)
with tf.variable_scope("Model_Encoder_Layer"):
attention_outputs = tf.contrib.layers.dropout(
attention_outputs, self.dropout, is_training=self.is_training)
# attention_outputs = conv_utils.linear_mapping(attention_outputs, self.hidden_size,
# dropout=self.dropout,
# var_scope_name="linear_mapping_before_rnn")
# attention_outputs = transformer.normalize(attention_outputs)
model_outputs, _ = layers.dynamic_origin_bilstm_layer(
attention_outputs,
self.hidden_size,
'model_layer',
input_len=self.frame_len)
model_outputs = tf.contrib.layers.dropout(
model_outputs, self.dropout, is_training=self.is_training)
with tf.variable_scope("Output_Layer"):
logit_score = layers.linear_layer_3d(model_outputs,
1,
scope_name='output_layer')
logit_score = tf.squeeze(logit_score, 2)
logit_loss = tf.nn.sigmoid_cross_entropy_with_logits(
logits=logit_score, labels=self.gt_predict)
avg_logit_loss = tf.reduce_mean(tf.reduce_sum(logit_loss, 1))
self.G_variables = tf.trainable_variables()
G_regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in self.G_variables])
self.test_loss = avg_logit_loss
self.loss = avg_logit_loss + self.regularization_beta * G_regularization_cost
self.frame_score = tf.nn.sigmoid(logit_score)
with tf.variable_scope('Pointer_Layer'):
score_dist = layers.linear_layer_3d(model_outputs,
2,
scope_name='pointer_layer')
predict_start = score_dist[:, :, 0]
predict_end = score_dist[:, :, 1]
start_loss = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=self.gt_start, logits=predict_start)
end_loss = tf.nn.softmax_cross_entropy_with_logits_v2(
labels=self.gt_end, logits=predict_end)
pointer_loss = tf.reduce_mean(start_loss + end_loss)
all_variable = tf.trainable_variables()
self.pn_variables = all_variable
# self.pn_variables = [vv for vv in all_variable if vv not in self.G_variables]
pn_regularization_cost = tf.reduce_sum(
[tf.nn.l2_loss(v) for v in self.pn_variables])
self.pn_loss = pointer_loss + self.regularization_beta * pn_regularization_cost
self.start_and_end = score_dist