def __init__(self, text_list, dict_path, vocab_dict, random = False,\ maxlen = 20, embedding_size = 128, **kwargs): super(WordEmbedding, self).__init__(**kwargs) self.embedding_path = kwargs['conf']['word_embedding_path'] self.vocab_dict = vocab_dict self.maxlen = maxlen self.dict_path = dict_path self.size = embedding_size self.trainable = kwargs['conf'].get('embedding_trainable', True) if random: self.embedding = tf.get_variable( "embeddings", shape=[len(self.vocab_dict), self.size], initializer=get_initializer('xavier'), trainable=self.trainable) else: loaded_embedding = self._get_embedding(self.vocab_dict) self.embedding = tf.get_variable( "embeddings", shape=[len(self.vocab_dict), self.size], initializer=get_initializer('xavier'), trainable=self.trainable) tf.assign(self.embedding, loaded_embedding) self.input_ids = {}
def __init__(self, text_list, dict_path, vocab_dict=None, random=False, maxlen=40, embedding_size=128, **kwargs): self.embedding_path = kwargs['conf']['char_embedding_path'] self.dict_path = dict_path self.maxlen = maxlen self.size = embedding_size self.vocab_dict = vocab_dict if random: self.embedding = tf.get_variable( "embeddings", shape=[len(self.vocab_dict), self.size], initializer=get_initializer('xavier'), trainable=True) else: loaded_embedding = self._get_embedding(self.vocab_dict) self.embedding = tf.get_variable( "embeddings", shape=[len(self.vocab_dict), self.size], initializer=get_initializer('xavier'), trainable=True) tf.assign(self.embedding, loaded_embedding) self.input_ids = {}
def __init__(self, **kwargs): super(DCNN, self).__init__(**kwargs) self.embed_dim = kwargs['embedding_size'] self.num_filters = [6, 14] self.top_k = 4 self.k1 = 19 self.num_hidden = 100 self.ws = [7, 5] self.W1 = tf.get_variable( "W1", [self.ws[0], int(self.embed_dim), 1, self.num_filters[0]], initializer=get_initializer(type='truncated_normal', stddev=0.01)) self.b1 = tf.get_variable("b1", [self.num_filters[0], self.embed_dim], initializer=get_initializer(type='constant', value=0.1)) self.W2 = tf.get_variable("W2", [ self.ws[1], int(self.embed_dim / 2), self.num_filters[0], self.num_filters[1] ], initializer=get_initializer( type='truncated_normal', stddev=0.01)) self.b2 = tf.get_variable("b2", [self.num_filters[1], self.embed_dim], initializer=get_initializer(type='constant', value=0.1))
def __init__(self, **args): self.batch_size = args['batch_size'] self.sentence_length = args['maxlen'] self.embed_dim = args['embedding_size'] self.keep_prob = args['keep_prob'] self.num_output = args['num_output'] self.num_filters = [6, 14] self.top_k = 4 self.k1 = 19 self.num_hidden = 100 self.ws = [7, 5] self.W1 = tf.get_variable( "W1", [self.ws[0], int(self.embed_dim), 1, self.num_filters[0]], initializer=get_initializer(type='truncated_normal', stddev=0.01)) self.b1 = tf.get_variable("b1", [self.num_filters[0], self.embed_dim], initializer=get_initializer(type='constant', value=0.1)) self.W2 = tf.get_variable("W2", [ self.ws[1], int(self.embed_dim / 2), self.num_filters[0], self.num_filters[1] ], initializer=get_initializer( type='truncated_normal', stddev=0.01)) self.b2 = tf.get_variable("b2", [self.num_filters[1], self.embed_dim], initializer=get_initializer(type='constant', value=0.1))
def __call__(self, embed, name = 'encoder', features = None, reuse = tf.AUTO_REUSE, **kwargs): with tf.variable_scope("fast_text", reuse = reuse): #embed: [batch_size, self.maxlen, embedding_size] length = tf.placeholder(tf.int32, name=name + '_length',shape=[]) if features != None: length = features[name + '_length'] #pdb.set_trace() #label_embedding: [num_output, num_hidden] label_embedding = tf.get_variable( "label_embedding", [self.num_output, self.num_hidden], initializer=get_initializer(type = 'random_uniform', minval = -0.01, maxval = 0.01) ) #mask:[batch_size, self.maxlen] mask = tf.sequence_mask(length, self.maxlen, tf.float32) mask = tf.expand_dims(mask, -1) embed = embed*mask mean_sentence = tf.reduce_mean(embed, axis=1) #vec: [batch_size, num_hidden] vec = tf.layers.dense(mean_sentence, self.num_hidden, kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001), name='fc', reuse = reuse) #att: [batch_size, num_output] att = tf.matmul(vec, tf.transpose(label_embedding)) return att
def __call__(self, embed, name='encoder', reuse=tf.AUTO_REUSE, **kwargs): #input: [batch_size, sentence_len, embedding_size,1] #output:[batch_size, num_output] with tf.variable_scope("text_cnn", reuse=reuse): embed = tf.expand_dims(embed, -1) conv_outputs = [] for i, size in enumerate(self.filter_sizes): with tf.variable_scope("conv%d" % i, reuse=reuse): # Convolution Layer begins conv_filter = tf.get_variable( "conv_filter%d" % i, [size, self.embedding_size, 1, self.num_filters], initializer=get_initializer(type='random_uniform', minval=-0.01, maxval=0.01)) bias = tf.get_variable( "conv_bias%d" % i, [self.num_filters], initializer=get_initializer(type='zeros')) output = tf.nn.conv2d(embed, conv_filter, [1, 1, 1, 1], "VALID") + bias # Applying non-linearity output = tf.nn.relu(output) # Pooling layer, max over time for each channel output = tf.reduce_max(output, axis=[1, 2]) conv_outputs.append(output) # Concatenate all different filter outputs before fully connected layers conv_outputs = tf.concat(conv_outputs, axis=1) #total_channels = conv_outputs.get_shape()[-1] h_pool_flat = tf.reshape( conv_outputs, [-1, self.num_filters * len(self.filter_sizes)]) h_drop = tf.nn.dropout(h_pool_flat, self.keep_prob) #dense = tf.layers.dense(h_drop, self.num_output, activation=None) dense = tf.layers.dense( h_pool_flat, self.num_output, kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001), activation=None, reuse=reuse) #logits = tf.layers.dense(mean_sentence, # self.num_output, # kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001), # name='fc', # reuse = reuse) return dense
def __init__(self, text_list, dict_path, vocab_dict, random = False,\ maxlen = 20, embedding_size = 128, **kwargs): self.embedding_path = kwargs['conf']['subword_embedding_path'] self.maxlen = maxlen self.dict_path = dict_path self.size = embedding_size self.embedding = tf.get_variable("embeddings", shape=[len(vocab_dict), self.size], initializer=get_initializer('xavier'), trainable=True) self.batch_size = kwargs['batch_size'] self.indices = {} self.values = {} self.input_ids = {}
def project_bilstm_layer(self, lstm_outputs, name=None): """ hidden layer between lstm layer and logits return: [batch_size, maxlen, num_output] """ with tf.variable_scope("project" if not name else name): outputs_shape = lstm_outputs.shape.as_list() with tf.variable_scope("hidden"): W = tf.get_variable("W", shape=[outputs_shape[-1], self.num_hidden], dtype=tf.float32, initializer=get_initializer(type='xavier')) b = tf.get_variable("b", shape=[self.num_hidden], dtype=tf.float32, initializer=tf.zeros_initializer()) output = tf.reshape(lstm_outputs, shape=[-1, outputs_shape[-1]]) hidden = tf.tanh(tf.nn.xw_plus_b(output, W, b)) # project to score of tags with tf.variable_scope("logits"): W = tf.get_variable("W", shape=[self.num_hidden, self.num_output], dtype=tf.float32, initializer=get_initializer(type='xavier')) b = tf.get_variable("b", shape=[self.num_output], dtype=tf.float32, initializer=tf.zeros_initializer()) pred = tf.nn.xw_plus_b(hidden, W, b) return tf.reshape(pred, [-1, self.maxlen, self.num_output])
def __call__(self, embed, name = 'encoder', reuse = tf.AUTO_REUSE): with tf.variable_scope("fast_text", reuse = reuse): att = tf.get_variable( "att", [self.maxlen, self.embedding_dim], initializer=get_initializer(minval = -0.01, maxval = 0.01) ) mul =tf.nn.softmax(tf.multiply(embed, att, name= 'attention_score')) embed = tf.multiply(embed, mul) mean_sentence = tf.reduce_mean(embed, axis=1) logits = tf.layers.dense(mean_sentence, self.hidden_num, kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001), name='fc') h_drop = tf.nn.dropout(logits, self.keep_prob) dense = tf.layers.dense(h_drop, self.num_output, activation=None) return dense
def CNN_layer(self, variable_scope, x1, x2, d, reuse): # x1, x2 = [batch, d, s, 1] with tf.variable_scope(variable_scope, reuse = reuse): if self.model_type == "ABCNN1" or self.model_type == "ABCNN3": with tf.name_scope("att_mat"): #[s, d] aW = tf.get_variable(name="aW", shape=(self.s, d), initializer=get_initializer(type='xavier'), regularizer=tf.contrib.layers.l2_regularizer(scale=self.l2_reg)) #[batch, s, s] att_mat = self.make_attention_mat(x1, x2) x1_a = \ tf.expand_dims(tf.matrix_transpose(tf.einsum("ijk,kl->ijl", att_mat, aW)), -1) x2_a = tf.expand_dims(tf.matrix_transpose( tf.einsum("ijk,kl->ijl", tf.matrix_transpose(att_mat), aW)), -1) # x1_a: [batch, d, s, 1] # x2_a: [batch, d, s, 1] # [batch, d, s, 2] x1 = tf.concat([x1, x1_a], axis=3) x2 = tf.concat([x2, x2_a], axis=3) left_conv = self.convolution(name_scope="left", x=self.pad_for_wide_conv(x1), d=d, reuse=False) right_conv = self.convolution(name_scope="right", x=self.pad_for_wide_conv(x2), d=d, reuse=True) left_attention, right_attention = None, None if self.model_type == "ABCNN2" or self.model_type == "ABCNN3": # [batch, s+w-1, s+w-1] att_mat = self.make_attention_mat(left_conv, right_conv) # [batch, s+w-1], [batch, s+w-1] left_attention, right_attention = tf.reduce_sum(att_mat, axis=2), tf.reduce_sum(att_mat, axis=1) left_wp = self.w_pool(variable_scope="left", x=left_conv, attention=left_attention, reuse = reuse) left_ap = self.all_pool(variable_scope="left", x=left_conv, reuse = reuse) right_wp = self.w_pool(variable_scope="right", x=right_conv, attention=right_attention, reuse = reuse) right_ap = self.all_pool(variable_scope="right", x=right_conv, reuse = reuse) return left_wp, left_ap, right_wp, right_ap