def layer_embedding(self): """ Layer embedding Default fuse word and char embeddings """ self.global_step = tf.Variable(0, name="global_step", trainable=False) with tf.variable_scope('embeddings'): self.word_mat = tf.get_variable( 'word_embeddings', shape=[self.vocab.word_size(), self.vocab.word_embed_dim], initializer=tf.constant_initializer( self.vocab.word_embeddings), trainable=False) self.char_mat = tf.get_variable( 'char_embeddins', shape=[self.vocab.char_size(), self.vocab.char_embed_dim], initializer=tf.constant_initializer( self.vocab.char_embeddings), trainable=False) sh_emb = tf.reshape(tf.nn.embedding_lookup( self.char_mat, self.sh), [ self.config.batch_size * self.config.max_sent_len, self.config.max_char_len, self.vocab.char_embed_dim ]) # projection sh_emb = Conv()(sh_emb, self.config.num_units, scope="sh_dense") sh_emb = tf.reduce_max(sh_emb, axis=1) sh_emb = tf.reshape( sh_emb, [self.config.batch_size, self.config.max_sent_len, -1]) s_emb = tf.nn.embedding_lookup(self.word_mat, self.s) s_emb += libs.position_embedding(self.s, self.vocab.word_embed_dim) s_emb = tf.concat([s_emb, sh_emb], axis=2) # projec self.s_emb = Conv()(s_emb, self.config.num_units, scope="s_proj") # self.s_emb = tf.layers.dropout(self.s_emb, 1.0-self.dropout) self.s_emb = Highway(activation=tf.nn.relu, kernel='conv', dropout=self.dropout)(self.s_emb, scope='highway')
def _conv(self, inputs): with tf.variable_scope(self.scope, reuse=self.reuse): size = inputs.shape.as_list()[-1] inputs = Conv()(inputs, size, scope="input_projection") for i in range(self.num_layers): H = Conv(activation=self.activation, bias=True)(inputs, size, scope="activation_%d" % i) T = Conv(bias=True, activation=tf.sigmoid)(inputs, size, scope="gate_%d" % i) H = tf.nn.dropout(H, 1.0 - self.dropout) inputs = H * T + inputs * (1.0 - T) return inputs
def layer_embedding(self): """ Layer embedding Default fuse word and char embeddings """ with tf.variable_scope('embeddings'): self.word_mat = tf.get_variable( 'word_embeddings', shape=[self.vocab.word_size(), self.vocab.word_embed_dim], initializer=tf.constant_initializer( self.vocab.word_embeddings), trainable=False) self.char_mat = tf.get_variable( 'char_embeddins', shape=[self.vocab.char_size(), self.vocab.char_embed_dim], initializer=tf.constant_initializer( self.vocab.char_embeddings), trainable=False) sh_emb = tf.reshape(tf.nn.embedding_lookup( self.char_mat, self.sh), [ self.config.batch_size * self.config.max_sent_len, self.config.max_char_len, self.vocab.char_embed_dim ]) # projection sh_emb = Conv(bias=True)(sh_emb, self.config.filter_size, scope="sh_conv") sh_emb = tf.reduce_max(sh_emb, axis=1) sh_emb = tf.reshape( sh_emb, [self.config.batch_size, self.config.max_sent_len, -1]) s_emb = tf.nn.embedding_lookup(self.word_mat, self.s) s_emb = tf.concat([s_emb, sh_emb], axis=2) # projection s_emb = Conv(bias=True)(s_emb, self.config.filter_size, scope="s_proj") self.s_emb = Highway(activation=tf.nn.relu, kernel='conv', dropout=self.dropout)(s_emb, scope='highway')
def layer_encoder(self): """ Layer Encoder Multi-channels convolution to encode """ with tf.variable_scope('encoder'): inputs = self.s_emb conv1 = Conv(bias=True, activation=tf.nn.relu, kernel_size=3)(inputs, self.config.filter_size, scope='conv1') conv2 = Conv(bias=True, activation=tf.nn.relu, kernel_size=3)(conv1, self.config.filter_size, scope='conv2') # shortcut conv2 = conv2 + inputs inputs = conv2 for i in range(self.config.num_blocks): seq_len = inputs.get_shape()[1] poolings = tf.transpose( tf.nn.top_k(tf.transpose(inputs, [0, 2, 1]), k=seq_len // 2)[0], [0, 2, 1]) conv1 = Conv(bias=True, activation=tf.nn.relu, kernel_size=3)(poolings, self.config.filter_size, scope='conv1-%d' % i) conv2 = Conv(bias=True, activation=tf.nn.relu, kernel_size=3)(conv1, self.config.filter_size, scope='conv2-%d' % i) inputs = conv2 + poolings self.output = tf.reduce_max(inputs, axis=1)