def _encode(self):
N, PL, QL, CL, d, dc, nh = self._params()
if self.config.fix_pretrained_vector:
dc = self.char_mat.get_shape()[-1]
with tf.variable_scope("Embedding_Encoder_Layer"):
self.c_embed_encoding = residual_block(self.c_emb,
num_blocks = 1,
num_conv_layers = 2,
kernel_size = 7,
mask = self.c_mask,
num_filters = d,
num_heads = nh,
seq_len = self.c_len,
scope = "Encoder_Residual_Block",
bias = False,
dropout = self.dropout)
self.q_embed_encoding = residual_block(self.q_emb,
num_blocks = 1,
num_conv_layers = 2,
kernel_size = 7,
mask = self.q_mask,
num_filters = d,
num_heads = nh,
seq_len = self.q_len,
scope = "Encoder_Residual_Block",
reuse = True, # Share the weights between passage and question
bias = False,
dropout = self.dropout)
def _fuse(self):
with tf.variable_scope("Context_to_Query_Attention_Layer"):
C = tf.tile(tf.expand_dims(self.c_embed_encoding, 2),
[1, 1, self.max_q_len, 1])
Q = tf.tile(tf.expand_dims(self.q_embed_encoding, 1),
[1, self.max_p_len, 1, 1])
S = trilinear([C, Q, C * Q], input_keep_prob=1.0 - self.dropout)
mask_q = tf.expand_dims(self.q_mask, 1)
S_ = tf.nn.softmax(mask_logits(S, mask=mask_q))
mask_c = tf.expand_dims(self.c_mask, 2)
S_T = tf.transpose(
tf.nn.softmax(mask_logits(S, mask=mask_c), dim=1), (0, 2, 1))
self.c2q = tf.matmul(S_, self.q_embed_encoding)
self.q2c = tf.matmul(tf.matmul(S_, S_T), self.c_embed_encoding)
self.attention_outputs = [
self.c_embed_encoding, self.c2q,
self.c_embed_encoding * self.c2q,
self.c_embed_encoding * self.q2c
]
N, PL, QL, CL, d, dc, nh = self._params()
if self.config.fix_pretrained_vector:
dc = self.char_mat.get_shape()[-1]
with tf.variable_scope("Model_Encoder_Layer"):
inputs = tf.concat(self.attention_outputs, axis=-1)
self.enc = [conv(inputs, d, name="input_projection")]
for i in range(3):
if i % 2 == 0:
self.enc[i] = tf.nn.dropout(self.enc[i],
1.0 - self.dropout)
self.enc.append(
residual_block(self.enc[i],
num_blocks=1,
num_conv_layers=2,
kernel_size=5,
mask=self.c_mask,
num_filters=d,
num_heads=nh,
seq_len=self.c_len,
scope="Model_Encoder",
bias=False,
reuse=True if i > 0 else None,
dropout=self.dropout))
for i, item in enumerate(self.enc):
self.enc[i] = tf.reshape(self.enc[i],
[N, -1, self.enc[i].get_shape()[-1]])