def _add_embedding_lookup(self):
with tf.variable_scope('word_embeddings'):
if self.cfg.use_word_emb:
_word_emb = tf.Variable(self.cfg.word_emb, name='_word_emb', trainable=self.cfg.finetune_emb,
dtype=tf.float32)
else:
_word_emb = tf.get_variable(name='_word_emb', shape=[self.cfg.vocab_size, self.cfg.word_dim],
trainable=True, dtype=tf.float32)
word_emb = tf.nn.embedding_lookup(_word_emb, self.word_ids, name='word_emb')
if self.cfg.use_char_emb: # use cnn to generate chars representation
with tf.variable_scope('char_embeddings'):
_char_emb = tf.get_variable(name='_char_emb', dtype=tf.float32, trainable=True,
shape=[self.cfg.char_vocab_size, self.cfg.char_dim])
char_emb = tf.nn.embedding_lookup(_char_emb, self.char_ids, name='char_emb')
char_emb_shape = tf.shape(char_emb)
char_rep = multi_conv1d(char_emb, self.cfg.filter_sizes, self.cfg.heights, "VALID", self.is_train,
self.cfg.keep_prob, scope="char_cnn")
char_rep = tf.reshape(char_rep, [char_emb_shape[0], char_emb_shape[1], self.cfg.char_rep_dim])
word_emb = tf.concat([word_emb, char_rep], axis=-1) # concat word emb and corresponding char rep
if self.cfg.use_highway:
self.word_emb = highway_network(word_emb, self.cfg.highway_num_layers, bias=True, is_train=self.is_train,
keep_prob=self.cfg.keep_prob)
else:
self.word_emb = dropout(word_emb, keep_prob=self.cfg.keep_prob, is_train=self.is_train)
print('word embedding shape: {}'.format(self.word_emb.get_shape().as_list()))
def _build_model(self):
with tf.variable_scope('dense_connect_bi_lstm'):
# create dense connected bi-lstm layers
dense_bi_lstm = []
for idx in range(self.cfg.num_layers):
if idx < self.cfg.num_layers - 1:
dense_bi_lstm.append(BiRNN(num_units=self.cfg.num_units, scope='bi_lstm_layer_{}'.format(idx)))
else:
dense_bi_lstm.append(BiRNN(num_units=self.cfg.num_units_last, scope='bi_lstm_layer_{}'.format(idx)))
# processing data
cur_inputs = self.word_emb
for idx in range(self.cfg.num_layers):
cur_rnn_outputs = dense_bi_lstm[idx](cur_inputs, seq_len=self.seq_len)
if idx < self.cfg.num_layers - 1:
cur_inputs = tf.concat([cur_inputs, cur_rnn_outputs], axis=-1)
else:
cur_inputs = cur_rnn_outputs
dense_bi_lstm_outputs = cur_inputs
print('dense bi-lstm outputs shape: {}'.format(dense_bi_lstm_outputs.get_shape().as_list()))
with tf.variable_scope('average_pooling'):
# according to the paper (https://arxiv.org/pdf/1802.00889.pdf) description in P4, simply compute average ?
avg_pool_outputs = tf.reduce_mean(dense_bi_lstm_outputs, axis=1)
avg_pool_outputs = dropout(avg_pool_outputs, keep_prob=self.cfg.keep_prob, is_train=self.is_train)
print('average pooling outputs shape: {}'.format(avg_pool_outputs.get_shape().as_list()))
with tf.variable_scope('project', regularizer=tf.contrib.layers.l2_regularizer(self.cfg.l2_reg)):
self.logits = dense(avg_pool_outputs, self.cfg.label_size, use_bias=True, scope='dense')
print('logits shape: {}'.format(self.logits.get_shape().as_list()))
def _build_embeddings_op(self):
with tf.variable_scope('words'):
if self.cfg.use_pretrained:
_word_embeddings = tf.Variable(self.cfg.glove_embeddings,
name='_word_embeddings',
dtype=tf.float32,
trainable=self.cfg.finetune_emb)
else:
_word_embeddings = tf.get_variable(
name='_word_embeddings',
dtype=tf.float32,
trainable=True,
shape=[self.cfg.word_vocab_size, self.cfg.word_dim])
word_embeddings = tf.nn.embedding_lookup(_word_embeddings,
self.word_ids,
name="word_embeddings")
with tf.variable_scope('char_represent'):
if self.cfg.use_char_emb:
_char_embeddings = tf.get_variable(
name='_char_embeddings',
dtype=tf.float32,
trainable=True,
shape=[self.cfg.char_vocab_size, self.cfg.char_dim])
char_embeddings = tf.nn.embedding_lookup(
_char_embeddings, self.char_ids, name="char_embeddings")
s = tf.shape(
char_embeddings
) # [batch size, max length of sentence, max length of word, char_dim]
output = multi_conv1d(char_embeddings,
self.cfg.filter_sizes,
self.cfg.heights,
"VALID",
self.is_train,
self.keep_prob,
scope="char_cnn")
# shape = (batch size, max sentence length, char representation size)
self.char_output = tf.reshape(
output, [s[0], s[1], self.cfg.char_out_size])
word_embeddings = tf.concat(
[word_embeddings, self.char_output], axis=-1)
if self.cfg.use_highway:
with tf.variable_scope("highway"):
self.word_embeddings = highway_network(
word_embeddings,
self.cfg.highway_num_layers,
bias=True,
is_train=self.is_train,
keep_prob=self.keep_prob)
else: # directly dropout before model_op
self.word_embeddings = dropout(word_embeddings,
keep_prob=self.keep_prob,
is_train=self.is_train)
print('word embeddings shape: {}'.format(
self.word_embeddings.get_shape().as_list()))
def _add_embedding_lookup(self):
with tf.variable_scope('word_embeddings'):
if self.cfg.use_word_emb:
_word_emb = tf.Variable(self.cfg.word_emb,
name='_word_emb',
trainable=self.cfg.finetune_emb,
dtype=tf.float32)
else:
_word_emb = tf.get_variable(
name='_word_emb',
shape=[self.cfg.vocab_size, self.cfg.word_dim],
trainable=True,
dtype=tf.float32)
word_emb = tf.nn.embedding_lookup(_word_emb,
self.word_ids,
name='word_emb')
if self.cfg.use_char_emb: # use cnn to generate chars representation
with tf.variable_scope('char_embeddings'):
_char_emb = tf.get_variable(
name='_char_emb',
dtype=tf.float32,
trainable=True,
shape=[self.cfg.char_vocab_size, self.cfg.char_dim])
char_emb = tf.nn.embedding_lookup(_char_emb,
self.char_ids,
name='char_emb')
char_emb_shape = tf.shape(char_emb)
char_rep = multi_conv1d(char_emb,
self.cfg.filter_sizes,
self.cfg.heights,
"VALID",
self.is_train,
self.cfg.keep_prob,
scope="char_cnn")
char_rep = tf.reshape(char_rep, [
char_emb_shape[0], char_emb_shape[1], self.cfg.char_rep_dim
])
word_emb = tf.concat(
[word_emb, char_rep],
axis=-1) # concat word emb and corresponding char rep
if self.cfg.use_highway:
self.word_emb = highway_network(word_emb,
self.cfg.highway_num_layers,
bias=True,
is_train=self.is_train,
keep_prob=self.cfg.keep_prob)
else:
self.word_emb = dropout(word_emb,
keep_prob=self.cfg.keep_prob,
is_train=self.is_train)
print('word embedding shape: {}'.format(
self.word_emb.get_shape().as_list()))
def _build_model(self):
with tf.variable_scope('dense_connect_bi_lstm'):
# create dense connected bi-lstm layers
dense_bi_lstm = [] #里面保存的都是一个个 双向的RNN的实例
for idx in range(self.cfg.num_layers):
if idx < self.cfg.num_layers - 1:
dense_bi_lstm.append(
BiRNN(num_units=self.cfg.num_units,
scope='bi_lstm_layer_{}'.format(idx)))
else:
dense_bi_lstm.append(
BiRNN(num_units=self.cfg.num_units_last,
scope='bi_lstm_layer_{}'.format(idx)))
# processing data
cur_inputs = self.word_emb
for idx in range(self.cfg.num_layers):
cur_rnn_outputs = dense_bi_lstm[idx](cur_inputs,
seq_len=self.seq_len)
if idx < self.cfg.num_layers - 1:
cur_inputs = tf.concat([cur_inputs, cur_rnn_outputs],
axis=-1) #这一步写的很好 运用的很灵活
else:
cur_inputs = cur_rnn_outputs #到了最后哦一层将不再进行拼接
dense_bi_lstm_outputs = cur_inputs
print('dense bi-lstm outputs shape: {}'.format(
dense_bi_lstm_outputs.get_shape().as_list()))
#将所有的时间步骤进行平均
with tf.variable_scope('average_pooling'):
# according to the paper (https://arxiv.org/pdf/1802.00889.pdf) description in P4, simply compute average ?
avg_pool_outputs = tf.reduce_mean(dense_bi_lstm_outputs, axis=1)
avg_pool_outputs = dropout(avg_pool_outputs,
keep_prob=self.cfg.keep_prob,
is_train=self.is_train)
# 这里猜测很简单 就是简单的 [None,100]
print('average pooling outputs shape: {}'.format(
avg_pool_outputs.get_shape().as_list()))
with tf.variable_scope('project',
regularizer=tf.contrib.layers.l2_regularizer(
self.cfg.l2_reg)):
self.logits = dense(avg_pool_outputs,
self.cfg.label_size,
use_bias=True,
scope='dense')
print('logits shape: {}'.format(self.logits.get_shape().as_list()))
def _build_embeddings_op(self):
with tf.variable_scope('words'):
if self.cfg.use_pretrained:
_word_embeddings = tf.Variable(self.cfg.glove_embeddings,
name='_word_embeddings',
dtype=tf.float32,
trainable=self.cfg.finetune_emb)
else:
_word_embeddings = tf.get_variable(
name='_word_embeddings',
dtype=tf.float32,
trainable=True,
shape=[self.cfg.word_vocab_size, self.cfg.word_dim])
word_embeddings = tf.nn.embedding_lookup(_word_embeddings,
self.word_ids,
name="word_embeddings")
with tf.variable_scope('char_rep_method'):
if self.cfg.use_char_emb:
_char_embeddings = tf.get_variable(
name='_char_embeddings',
dtype=tf.float32,
trainable=True,
shape=[self.cfg.char_vocab_size, self.cfg.char_dim])
char_embeddings = tf.nn.embedding_lookup(
_char_embeddings, self.char_ids, name="char_embeddings")
s = tf.shape(
char_embeddings
) # [batch size, max length of sentence, max length of word, char_dim]
if self.cfg.char_rep_method == 'rnn':
char_embeddings = tf.reshape(
char_embeddings,
shape=[s[0] * s[1], s[-2], self.cfg.char_dim])
word_lengths = tf.reshape(self.word_lengths,
shape=[s[0] * s[1]])
char_bi_rnn = BiRNN(self.cfg.num_units_char,
scope='char_rnn')
output = char_bi_rnn(char_embeddings,
word_lengths,
return_last_state=True)
else: # cnn model for char representation
output = multi_conv1d(char_embeddings,
self.cfg.filter_sizes,
self.cfg.heights,
"VALID",
self.is_train,
self.keep_prob,
scope="char_cnn")
# shape = (batch size, max sentence length, char representation size)
self.char_output = tf.reshape(
output, [s[0], s[1], self.cfg.char_out_size])
word_embeddings = tf.concat(
[word_embeddings, self.char_output], axis=-1)
if self.cfg.use_highway:
with tf.variable_scope("highway"):
self.word_embeddings = highway_network(
word_embeddings,
self.cfg.highway_num_layers,
bias=True,
is_train=self.is_train,
keep_prob=self.keep_prob)
else: # directly dropout before model_op
self.word_embeddings = dropout(word_embeddings,
keep_prob=self.keep_prob,
is_train=self.is_train)
def _add_embedding_lookup(self):
with tf.variable_scope('word_embeddings'):
if self.cfg.use_word_emb:
#注意这种用法 声明具有初始值的变量
_word_emb = tf.Variable(self.cfg.word_emb,
name='_word_emb',
trainable=self.cfg.finetune_emb,
dtype=tf.float32)
else:
#声明一般变量
_word_emb = tf.get_variable(
name='_word_emb',
shape=[self.cfg.vocab_size, self.cfg.word_dim],
trainable=True,
dtype=tf.float32)
word_emb = tf.nn.embedding_lookup(_word_emb,
self.word_ids,
name='word_emb')
if self.cfg.use_char_emb: # use cnn to generate chars representation
with tf.variable_scope('char_embeddings'):
_char_emb = tf.get_variable(
name='_char_emb',
dtype=tf.float32,
trainable=True,
shape=[self.cfg.char_vocab_size, self.cfg.char_dim])
#在理解的时候采用一个样本进行理解
char_emb = tf.nn.embedding_lookup(
_char_emb, self.char_ids, name='char_emb'
) # 这里我要搞清楚这个更！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！！
print("许海明的测试-------》char_emb的形状",
char_emb.get_shape().as_list())
#这一步类似于一个图片 不过不是RGB 而是一句话最长的单词的数目
char_emb_shape = tf.shape(char_emb) #这里需要注意一下 这里的形状要搞清楚
# [-1,max_len_sen,char_out_size*len([fileter])]
char_rep = multi_conv1d(char_emb,
self.cfg.filter_sizes,
self.cfg.heights,
"VALID",
self.is_train,
self.cfg.keep_prob,
scope="char_cnn")
char_rep = tf.reshape(char_rep, [
char_emb_shape[0], char_emb_shape[1], self.cfg.char_rep_dim
])
word_emb = tf.concat(
[word_emb, char_rep],
axis=-1) # concat word emb and corresponding char rep
if self.cfg.use_highway:
self.word_emb = highway_network(word_emb,
self.cfg.highway_num_layers,
bias=True,
is_train=self.is_train,
keep_prob=self.cfg.keep_prob)
else:
self.word_emb = dropout(word_emb,
keep_prob=self.cfg.keep_prob,
is_train=self.is_train)
print('word embedding shape: {}'.format(
self.word_emb.get_shape().as_list()))
import numpy as np
from scipy import spatial
import datetime
# Get starting time
start = datetime.datetime.now()
time = start.strftime("%m-%d-%H-%M-%S")
import models
model_dict = {
'base': models.base(),
'dropout': models.dropout(),
'multi': models.multi(),
}
import keras
from keras import backend as K
def euc_dist_keras(y_true, y_pred):
"""Loss function to be used in training."""
return K.sqrt(K.sum(K.square(y_true - y_pred), axis=-1, keepdims=True))
from keras.models import Model
from keras.layers import Dense, Activation, Dropout, Input
from keras.optimizers import SGD
from keras.callbacks import ModelCheckpoint
from keras.models import model_from_json
from keras.layers.merge import Concatenate, Add