def __init__(self, text_list, dict_path, vocab_dict, random = False,\
maxlen = 20, embedding_size = 128, **kwargs):
super(WordEmbedding, self).__init__(**kwargs)
self.embedding_path = kwargs['conf']['word_embedding_path']
self.vocab_dict = vocab_dict
self.maxlen = maxlen
self.dict_path = dict_path
self.size = embedding_size
self.trainable = kwargs['conf'].get('embedding_trainable', True)
if random:
self.embedding = tf.get_variable(
"embeddings",
shape=[len(self.vocab_dict), self.size],
initializer=get_initializer('xavier'),
trainable=self.trainable)
else:
loaded_embedding = self._get_embedding(self.vocab_dict)
self.embedding = tf.get_variable(
"embeddings",
shape=[len(self.vocab_dict), self.size],
initializer=get_initializer('xavier'),
trainable=self.trainable)
tf.assign(self.embedding, loaded_embedding)
self.input_ids = {}
def __init__(self,
text_list,
dict_path,
vocab_dict=None,
random=False,
maxlen=40,
embedding_size=128,
**kwargs):
self.embedding_path = kwargs['conf']['char_embedding_path']
self.dict_path = dict_path
self.maxlen = maxlen
self.size = embedding_size
self.vocab_dict = vocab_dict
if random:
self.embedding = tf.get_variable(
"embeddings",
shape=[len(self.vocab_dict), self.size],
initializer=get_initializer('xavier'),
trainable=True)
else:
loaded_embedding = self._get_embedding(self.vocab_dict)
self.embedding = tf.get_variable(
"embeddings",
shape=[len(self.vocab_dict), self.size],
initializer=get_initializer('xavier'),
trainable=True)
tf.assign(self.embedding, loaded_embedding)
self.input_ids = {}
def __init__(self, **kwargs):
super(DCNN, self).__init__(**kwargs)
self.embed_dim = kwargs['embedding_size']
self.num_filters = [6, 14]
self.top_k = 4
self.k1 = 19
self.num_hidden = 100
self.ws = [7, 5]
self.W1 = tf.get_variable(
"W1", [self.ws[0],
int(self.embed_dim), 1, self.num_filters[0]],
initializer=get_initializer(type='truncated_normal', stddev=0.01))
self.b1 = tf.get_variable("b1", [self.num_filters[0], self.embed_dim],
initializer=get_initializer(type='constant',
value=0.1))
self.W2 = tf.get_variable("W2", [
self.ws[1],
int(self.embed_dim / 2), self.num_filters[0], self.num_filters[1]
],
initializer=get_initializer(
type='truncated_normal', stddev=0.01))
self.b2 = tf.get_variable("b2", [self.num_filters[1], self.embed_dim],
initializer=get_initializer(type='constant',
value=0.1))
def __init__(self, **args):
self.batch_size = args['batch_size']
self.sentence_length = args['maxlen']
self.embed_dim = args['embedding_size']
self.keep_prob = args['keep_prob']
self.num_output = args['num_output']
self.num_filters = [6, 14]
self.top_k = 4
self.k1 = 19
self.num_hidden = 100
self.ws = [7, 5]
self.W1 = tf.get_variable(
"W1", [self.ws[0],
int(self.embed_dim), 1, self.num_filters[0]],
initializer=get_initializer(type='truncated_normal', stddev=0.01))
self.b1 = tf.get_variable("b1", [self.num_filters[0], self.embed_dim],
initializer=get_initializer(type='constant',
value=0.1))
self.W2 = tf.get_variable("W2", [
self.ws[1],
int(self.embed_dim / 2), self.num_filters[0], self.num_filters[1]
],
initializer=get_initializer(
type='truncated_normal', stddev=0.01))
self.b2 = tf.get_variable("b2", [self.num_filters[1], self.embed_dim],
initializer=get_initializer(type='constant',
value=0.1))
def __call__(self, embed, name = 'encoder', features = None, reuse = tf.AUTO_REUSE, **kwargs):
with tf.variable_scope("fast_text", reuse = reuse):
#embed: [batch_size, self.maxlen, embedding_size]
length = tf.placeholder(tf.int32, name=name + '_length',shape=[])
if features != None:
length = features[name + '_length']
#pdb.set_trace()
#label_embedding: [num_output, num_hidden]
label_embedding = tf.get_variable(
"label_embedding",
[self.num_output, self.num_hidden],
initializer=get_initializer(type = 'random_uniform',
minval = -0.01,
maxval = 0.01)
)
#mask:[batch_size, self.maxlen]
mask = tf.sequence_mask(length, self.maxlen, tf.float32)
mask = tf.expand_dims(mask, -1)
embed = embed*mask
mean_sentence = tf.reduce_mean(embed, axis=1)
#vec: [batch_size, num_hidden]
vec = tf.layers.dense(mean_sentence,
self.num_hidden,
kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001),
name='fc',
reuse = reuse)
#att: [batch_size, num_output]
att = tf.matmul(vec, tf.transpose(label_embedding))
return att
def __call__(self, embed, name='encoder', reuse=tf.AUTO_REUSE, **kwargs):
#input: [batch_size, sentence_len, embedding_size,1]
#output:[batch_size, num_output]
with tf.variable_scope("text_cnn", reuse=reuse):
embed = tf.expand_dims(embed, -1)
conv_outputs = []
for i, size in enumerate(self.filter_sizes):
with tf.variable_scope("conv%d" % i, reuse=reuse):
# Convolution Layer begins
conv_filter = tf.get_variable(
"conv_filter%d" % i,
[size, self.embedding_size, 1, self.num_filters],
initializer=get_initializer(type='random_uniform',
minval=-0.01,
maxval=0.01))
bias = tf.get_variable(
"conv_bias%d" % i, [self.num_filters],
initializer=get_initializer(type='zeros'))
output = tf.nn.conv2d(embed, conv_filter, [1, 1, 1, 1],
"VALID") + bias
# Applying non-linearity
output = tf.nn.relu(output)
# Pooling layer, max over time for each channel
output = tf.reduce_max(output, axis=[1, 2])
conv_outputs.append(output)
# Concatenate all different filter outputs before fully connected layers
conv_outputs = tf.concat(conv_outputs, axis=1)
#total_channels = conv_outputs.get_shape()[-1]
h_pool_flat = tf.reshape(
conv_outputs, [-1, self.num_filters * len(self.filter_sizes)])
h_drop = tf.nn.dropout(h_pool_flat, self.keep_prob)
#dense = tf.layers.dense(h_drop, self.num_output, activation=None)
dense = tf.layers.dense(
h_pool_flat,
self.num_output,
kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001),
activation=None,
reuse=reuse)
#logits = tf.layers.dense(mean_sentence,
# self.num_output,
# kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001),
# name='fc',
# reuse = reuse)
return dense
def __init__(self, text_list, dict_path, vocab_dict, random = False,\
maxlen = 20, embedding_size = 128, **kwargs):
self.embedding_path = kwargs['conf']['subword_embedding_path']
self.maxlen = maxlen
self.dict_path = dict_path
self.size = embedding_size
self.embedding = tf.get_variable("embeddings",
shape=[len(vocab_dict), self.size],
initializer=get_initializer('xavier'),
trainable=True)
self.batch_size = kwargs['batch_size']
self.indices = {}
self.values = {}
self.input_ids = {}
def project_bilstm_layer(self, lstm_outputs, name=None):
"""
hidden layer between lstm layer and logits
return: [batch_size, maxlen, num_output]
"""
with tf.variable_scope("project" if not name else name):
outputs_shape = lstm_outputs.shape.as_list()
with tf.variable_scope("hidden"):
W = tf.get_variable("W",
shape=[outputs_shape[-1], self.num_hidden],
dtype=tf.float32,
initializer=get_initializer(type='xavier'))
b = tf.get_variable("b",
shape=[self.num_hidden],
dtype=tf.float32,
initializer=tf.zeros_initializer())
output = tf.reshape(lstm_outputs,
shape=[-1, outputs_shape[-1]])
hidden = tf.tanh(tf.nn.xw_plus_b(output, W, b))
# project to score of tags
with tf.variable_scope("logits"):
W = tf.get_variable("W",
shape=[self.num_hidden, self.num_output],
dtype=tf.float32,
initializer=get_initializer(type='xavier'))
b = tf.get_variable("b",
shape=[self.num_output],
dtype=tf.float32,
initializer=tf.zeros_initializer())
pred = tf.nn.xw_plus_b(hidden, W, b)
return tf.reshape(pred, [-1, self.maxlen, self.num_output])
def __call__(self, embed, name = 'encoder', reuse = tf.AUTO_REUSE):
with tf.variable_scope("fast_text", reuse = reuse):
att = tf.get_variable(
"att", [self.maxlen, self.embedding_dim],
initializer=get_initializer(minval = -0.01, maxval = 0.01)
)
mul =tf.nn.softmax(tf.multiply(embed, att, name= 'attention_score'))
embed = tf.multiply(embed, mul)
mean_sentence = tf.reduce_mean(embed, axis=1)
logits = tf.layers.dense(mean_sentence,
self.hidden_num,
kernel_regularizer=tf.contrib.layers.l2_regularizer(0.001),
name='fc')
h_drop = tf.nn.dropout(logits, self.keep_prob)
dense = tf.layers.dense(h_drop, self.num_output, activation=None)
return dense
def CNN_layer(self, variable_scope, x1, x2, d, reuse):
# x1, x2 = [batch, d, s, 1]
with tf.variable_scope(variable_scope, reuse = reuse):
if self.model_type == "ABCNN1" or self.model_type == "ABCNN3":
with tf.name_scope("att_mat"):
#[s, d]
aW = tf.get_variable(name="aW",
shape=(self.s, d),
initializer=get_initializer(type='xavier'),
regularizer=tf.contrib.layers.l2_regularizer(scale=self.l2_reg))
#[batch, s, s]
att_mat = self.make_attention_mat(x1, x2)
x1_a = \
tf.expand_dims(tf.matrix_transpose(tf.einsum("ijk,kl->ijl",
att_mat, aW)), -1)
x2_a = tf.expand_dims(tf.matrix_transpose(
tf.einsum("ijk,kl->ijl", tf.matrix_transpose(att_mat),
aW)), -1)
# x1_a: [batch, d, s, 1]
# x2_a: [batch, d, s, 1]
# [batch, d, s, 2]
x1 = tf.concat([x1, x1_a], axis=3)
x2 = tf.concat([x2, x2_a], axis=3)
left_conv = self.convolution(name_scope="left", x=self.pad_for_wide_conv(x1), d=d, reuse=False)
right_conv = self.convolution(name_scope="right", x=self.pad_for_wide_conv(x2), d=d, reuse=True)
left_attention, right_attention = None, None
if self.model_type == "ABCNN2" or self.model_type == "ABCNN3":
# [batch, s+w-1, s+w-1]
att_mat = self.make_attention_mat(left_conv, right_conv)
# [batch, s+w-1], [batch, s+w-1]
left_attention, right_attention = tf.reduce_sum(att_mat, axis=2), tf.reduce_sum(att_mat, axis=1)
left_wp = self.w_pool(variable_scope="left", x=left_conv, attention=left_attention, reuse = reuse)
left_ap = self.all_pool(variable_scope="left", x=left_conv, reuse = reuse)
right_wp = self.w_pool(variable_scope="right", x=right_conv, attention=right_attention, reuse = reuse)
right_ap = self.all_pool(variable_scope="right", x=right_conv, reuse = reuse)
return left_wp, left_ap, right_wp, right_ap
