def _build_embed_encoder_and_attention(self, x, q, context_lengths, question_lengths): """Build modeling encoder and return start/end logits. Args: x: context Tensor of shape [batch, max_context_length, dim]. q: question Tensor of shape [batch, max_question_length, dim]. context_lengths: length Tensor of shape [batch]. question_lengths: length Tensor of shape [batch]. Returns: x_final: output Tensor of shape [batch, max_context_length, dim]. """ input_keep_prob = self.config['input_keep_prob'] # Embedding encoder encoder_emb = self.init_submodule( self.config['encoder_emb'], name='xq_encoder') x = encoder_emb(x, context_lengths)['outputs'] q = encoder_emb(q, question_lengths)['outputs'] x = squad_helper.dropout_wrapper(x, input_keep_prob, self.mode) q = squad_helper.dropout_wrapper(q, input_keep_prob, self.mode) # Context-Question Attention with tf.variable_scope('attention'): xq, qx = squad_helper.bi_attention_memory_efficient_dcn( a=q, b=x, mask_a=question_lengths, mask_b=context_lengths) x_final = tf.concat([x, xq, x * xq, x * qx], 2) return x_final
def _build_embed(self, features, training): """Build embedding model and return document/question representations. Args: features: dict; input data. training: bool; whether we are under training mode. Returns: x: context Tensor of shape [batch, max_context_length, dim]. q: question Tensor of shape [batch, max_question_length, dim]. """ config = self.config tf.logging.info('\n\n\n\n\n') tf.logging.info(config) tf.logging.info('\n\n\n\n\n') embedder = self.init_submodule(config['embedding']) xc, qc, xw, qw, _, _, _, _ = embedder(features) emb = embedder(features) # Concatenate word and character embeddings x = tf.concat([emb['xc'], emb['xw']], 2) q = tf.concat([emb['qc'], emb['qw']], 2) x = squad_helper.dropout_wrapper(x, config['input_keep_prob'], self.mode) q = squad_helper.dropout_wrapper(q, config['input_keep_prob'], self.mode) x = squad_helper.highway_layer( x, num_layers=2, activation='relu', training=training, dropout_rate=1-config['hw_keep_prob']) q = squad_helper.highway_layer( q, num_layers=2, activation='relu', training=training, dropout_rate=1-config['hw_keep_prob'], reuse=True) return x, q
def _build_model_encoder(self, x, context_lengths, gpu_id=0, num_gpus=1): """Build modeling encoder and return start/end logits. Args: x: input Tensor of shape [batch, max_length, dim]. context_lengths: length Tensor of shape [batch]. gpu_id: start GPU id. num_gpus: number of GPUs available. Returns: logits_start: Tensor of shape [batch, max_length]. logits_end: Tensor of shape [batch, max_length]. modeling_layers: a list of modeling layers, from bottom to top, each has shape [batch, max_length, dim]. """ output_keep_prob = self.config['output_keep_prob'] with tf.device(misc_util.get_device_str(gpu_id, num_gpus)): encoder_model = self.init_submodule( self.config['encoder_model'], name='encoder_model') x0 = encoder_model(x, context_lengths)['outputs'] x0 = squad_helper.dropout_wrapper(x0, output_keep_prob, self.mode) x1 = encoder_model(x0, context_lengths)['outputs'] x1 = squad_helper.dropout_wrapper(x1, output_keep_prob, self.mode) gpu_id += 1 with tf.device(misc_util.get_device_str(gpu_id, num_gpus)): x2 = encoder_model(x1, context_lengths)['outputs'] x2 = squad_helper.dropout_wrapper(x2, output_keep_prob, self.mode) x3 = encoder_model(x2, context_lengths)['outputs'] x3 = squad_helper.dropout_wrapper(x3, output_keep_prob, self.mode) logits_start = squad_helper.exp_mask( tf.squeeze( tf.layers.dense(tf.concat([x1, x2], 2), 1, name='logits1'), 2), context_lengths) logits_end = squad_helper.exp_mask( tf.squeeze( tf.layers.dense(tf.concat([x1, x3], 2), 1, name='logits2'), 2), context_lengths) modeling_layers = [x0, x1, x2, x3] return logits_start, logits_end, modeling_layers