def __init__(self, hidden_size, keep_prob, num_layers):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.num_layers = num_layers
self.rnn_cell_fw = [
rnn_cell.GRUCell(self.hidden_size) for _ in range(self.num_layers)
]
self.rnn_cell_fw = [
DropoutWrapper(cell, input_keep_prob=self.keep_prob)
for cell in self.rnn_cell_fw
]
self.multi_rnn_cell_fw = rnn_cell.MultiRNNCell(self.rnn_cell_fw,
state_is_tuple=False)
self.rnn_cell_bw = [
rnn_cell.GRUCell(self.hidden_size) for _ in range(self.num_layers)
]
self.rnn_cell_bw = [
DropoutWrapper(cell, input_keep_prob=self.keep_prob)
for cell in self.rnn_cell_bw
]
self.multi_rnn_cell_bw = rnn_cell.MultiRNNCell(self.rnn_cell_bw,
state_is_tuple=False)
def __init__(self,
hidden_size,
keep_prob,
cell_type="gru",
scope="encoder"):
"""
Inputs:
hidden_size:
int.
Hidden size of the RNN
keep_prob:
Tensor
containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
if cell_type == "gru":
rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size)
rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size)
elif cell_type == "lstm":
rnn_cell_fw = rnn_cell.LSTMCell(self.hidden_size)
rnn_cell_bw = rnn_cell.LSTMCell(self.hidden_size)
else:
assert (False, "No such cell type for RNN encoder!")
self.rnn_cell_fw = DropoutWrapper(rnn_cell_fw,
input_keep_prob=self.keep_prob)
self.rnn_cell_bw = DropoutWrapper(rnn_cell_bw,
input_keep_prob=self.keep_prob)
self.scope = scope
logger.info("Encoder created: {} | hidden_size = {}".format(
cell_type, hidden_size))
def BIGRU(x, hidden_size):
# biLSTM��
# ���ܣ����bidirectional_lstm����
# ������
# x: [batch, height, width] / [batch, step, embedding_size]
# hidden_size: lstm���ز�ڵ����
# �����
# output: [batch, height, 2*hidden_size] / [batch, step, 2*hidden_size]
# input transformation
input_x = tf.transpose(x, [1, 0, 2])
# input_x = tf.reshape(input_x, [-1, w])
# input_x = tf.split(0, h, input_x)
input_x = tf.unstack(input_x)
# define the forward and backward lstm cells
gru_fw_cell = rnn_cell.GRUCell(hidden_size)
gru_bw_cell = rnn_cell.GRUCell(hidden_size)
output, _, _ = tf.contrib.rnn.stack_bidirectional_rnn(gru_fw_cell,
gru_bw_cell,
input_x,
dtype=tf.float32)
# output transformation to the original tensor type
output = tf.stack(output)
output = tf.transpose(output, [1, 0, 2])
return output
def __init__(self, size, num_layers, tied_weights=True):
self.size = size
self.keep_prob = tf.placeholder(tf.float32)
self.tied_weights = tied_weights
# when we use different weights, we force the model to concatenate
if not self.tied_weights:
FLAGS.concat = True
if FLAGS.rnn == "lstm":
cell = rnn_cell.BasicLSTMCell(self.size)
state_is_tuple = True
else:
cell = rnn_cell.GRUCell(self.size)
state_is_tuple = False
cell = DropoutWrapper(cell, input_keep_prob=self.keep_prob, seed=123)
self.encoder_cell = tf.nn.rnn_cell.MultiRNNCell(
[cell] * num_layers, state_is_tuple=state_is_tuple)
if not tied_weights:
if FLAGS.rnn == "lstm":
cell_back = rnn_cell.BasicLSTMCell(self.size)
state_is_tuple = True
else:
cell_back = rnn_cell.GRUCell(self.size)
state_is_tuple = False
cell_back = DropoutWrapper(cell_back,
input_keep_prob=self.keep_prob,
seed=123)
self.encoder_cell_bw = tf.nn.rnn_cell.MultiRNNCell(
[cell_back] * num_layers, state_is_tuple=state_is_tuple)
def decode(self, decoder_inputs, encoder_output, target_length, reuse=False):
if self.num_layers > 1:
self.decoder_cell = rnn_cell.GRUCell(self.size)
if self.attn:
self.attn_cell = GRUCellAttn(self.size, encoder_output, scope="DecoderAttnCell")
else:
self.attn_cell = rnn_cell.GRUCell(self.size)
i = -1
with vs.variable_scope("Decoder", reuse=reuse):
inp = decoder_inputs
for i in xrange(self.num_layers - 1):
with vs.variable_scope("DecoderCell%d" % i) as scope:
out, state_output = rnn.dynamic_rnn(self.decoder_cell, inp, time_major=False,
dtype=tf.float32, sequence_length=target_length,
scope=scope, initial_state=self.decoder_state_input[i])
inp = self.dropout(out)
self.decoder_state_output.append(state_output)
with vs.variable_scope("DecoderAttnCell") as scope:
out, state_output = rnn.dynamic_rnn(self.attn_cell, inp, time_major=False,
dtype=tf.float32, sequence_length=target_length,
scope=scope, initial_state=self.decoder_state_input[i + 1])
decoder_output = self.dropout(out)
self.decoder_state_output.append(state_output)
# (batch_size, T, hidden_size)
return decoder_output
def setup_encoder(self): # encoder的设置
with vs.variable_scope("Encoder"): # 在encoder的作用域下
inp = tf.nn.dropout(
self.encoder_inputs, self.keep_prob
) # 对encoder进行dropout dropout率为 keep_prob -> 减少过拟合
fw_cell = rnn_cell.GRUCell(self.size) # GRU单元
fw_cell = rnn_cell.DropoutWrapper(
fw_cell, output_keep_prob=self.keep_prob) # 对单元进行dropout
self.encoder_fw_cell = rnn_cell.MultiRNNCell( # 创建多层RNN的函数 encoder 前向单元
[fw_cell] * self.num_layers,
state_is_tuple=True) # 设置multi-rnn cell
bw_cell = rnn_cell.GRUCell(self.size) # 设置size大小的GRU单元
bw_cell = rnn_cell.DropoutWrapper( # 根据dropout率，随机在抛弃GRU中计算的数据
bw_cell,
output_keep_prob=self.keep_prob)
self.encoder_bw_cell = rnn_cell.MultiRNNCell( # 设置 encoder 反向单元
[bw_cell] * self.num_layers,
state_is_tuple=True)
out, _ = rnn.bidirectional_dynamic_rnn(
self.encoder_fw_cell, # 设置动态双向RNN
self.encoder_bw_cell,
inp,
self.src_len,
dtype=tf.float32,
time_major=True,
initial_state_fw=self.encoder_fw_cell.zero_state(
self.batch_size, dtype=tf.float32), # 状态全部初始化为0
initial_state_bw=self.encoder_bw_cell.zero_state(
self.batch_size, dtype=tf.float32))
out = tf.concat([out[0], out[1]], axis=2) # 把 1 和 2拼接起来
self.encoder_output = out
def BILSTM(x, hidden_size):
# biLSTM：
# 功能：添加bidirectional_lstm操作
# 参数：
# x: [batch, height, width] / [batch, step, embedding_size]
# hidden_size: lstm隐藏层节点个数
# 输出：
# output: [batch, height, 2*hidden_size] / [batch, step, 2*hidden_size]
# input transformation
input_x = tf.transpose(x, [1, 0, 2])
# input_x = tf.reshape(input_x, [-1, w])
# input_x = tf.split(0, h, input_x)
input_x = tf.unpack(input_x)
# define the forward and backward lstm cells
gru_fw_cell = rnn_cell.GRUCell(hidden_size)
gru_bw_cell = rnn_cell.GRUCell(hidden_size)
output, _, _ = rnn.bidirectional_rnn(gru_fw_cell,
gru_bw_cell,
input_x,
dtype=tf.float32)
# output transformation to the original tensor type
output = tf.pack(output)
output = tf.transpose(output, [1, 0, 2])
return output
def __init__(self, hidden_size):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.rnn_cell_fw = tf.nn.rnn_cell.MultiRNNCell(
[rnn_cell.GRUCell(self.hidden_size) for i in range(2)])
self.rnn_cell_bw = tf.nn.rnn_cell.MultiRNNCell(
[rnn_cell.GRUCell(self.hidden_size) for i in range(2)])
def __init__(self, hidden_size, keep_prob):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob)
self.rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def BIGRU(x, hidden_size):
input_x = tf.transpose(x, [1, 0, 2])
input_x = tf.unstack(input_x)
# define the forward and backward lstm cells
gru_fw_cell = rnn_cell.GRUCell(hidden_size)
gru_bw_cell = rnn_cell.GRUCell(hidden_size)
output, _, _ = rnn.bidirectional_rnn(gru_fw_cell, gru_bw_cell, input_x, dtype=tf.float32)
# output transformation to the original tensor type
output = tf.stack(output)
output = tf.transpose(output, [1, 0, 2])
return output
def create_scopes():
self.descs = tf.constant(descs,
dtype=tf.float32,
shape=[self.num_types, self.num_types],
name='descs')
max_val = np.sqrt(6. / (self.num_types + self.num_types))
if self.attention_type == self.CHANGE_OF_VARIABLES:
self.attn_aggrW = tf.Variable(tf.random_uniform(
[self.num_types, self.num_types], -1. * max_val, max_val),
name="attn_aggrW")
self.type_embedding = tf.get_variable(
'type_embedding',
shape=[self.num_types, self.num_type_dim],
initializer=tf.truncated_normal_initializer(0.0, 1.0))
max_val = np.sqrt(6. / (self.hidden_size + self.num_type_dim))
self.type_context_scope = "type_context_scope"
self.type_context_W = tf.Variable(tf.random_uniform(
[self.hidden_size, self.num_type_dim], -1. * max_val, max_val),
name="W_type_context")
self.type_context_attnW = tf.Variable(tf.random_uniform(
[self.hidden_size, self.num_type_dim], -1. * max_val, max_val),
name="attnW_type_context")
self.type_mention_scope = "type_mention_scope"
self.type_mention_W = tf.Variable(tf.random_uniform(
[self.hidden_size, self.num_type_dim], -1. * max_val, max_val),
name="W_type_mention")
self.type_mention_attnW = tf.Variable(tf.random_uniform(
[self.hidden_size, self.num_type_dim], -1. * max_val, max_val),
name="attnW_type_mention")
self.enc_rnn_left_context = rnn_cell.EmbeddingWrapper(
rnn_cell.GRUCell(self.hidden_size), self.num_words,
self.num_word_dim)
self.enc_scope_left_context = "encoder_left_context"
self.enc_rnn_right_context = rnn_cell.EmbeddingWrapper(
rnn_cell.GRUCell(self.hidden_size), self.num_words,
self.num_word_dim)
self.enc_scope_right_context = "encoder_right_context"
self.enc_rnn_mention = rnn_cell.EmbeddingWrapper(
rnn_cell.GRUCell(self.hidden_size), self.num_mention_words,
self.num_mention_dim)
self.enc_scope_mention = "encoder_mention"
self.dec_cells = rnn_cell.GRUCell(self.hidden_size)
self.dec_scope = "decoder"
max_val = np.sqrt(6. / (self.num_types + self.hidden_size))
self.dec_weights = tf.get_variable(
"dec_weights", [self.hidden_size, self.num_type_dim],
initializer=tf.random_uniform_initializer(
-1. * max_val, max_val))
self.dec_biases = tf.get_variable(
"dec_biases", [self.num_type_dim],
initializer=tf.constant_initializer(0.0))
def __init__(self, hidden_size, keep_prob, attention_mechanism=None, name="RNNEncoder"):
with vs.variable_scope(name):
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size)
if attention_mechanism is not None:
self.rnn_cell_fw = AttentionWrapper(self.rnn_cell_fw, attention_mechanism)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob)
self.rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size)
if attention_mechanism is not None:
self.rnn_cell_bw = AttentionWrapper(self.rnn_cell_bw, attention_mechanism)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
self.name = name
def __init__(self, hidden_size, keep_prob):
"""
hidden_size: number of hidden size of RNN
keep_prob: keep prob. for dropout
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.fw_rnn = rnn_cell.GRUCell(self.hidden_size)
self.fw_rnn = DropoutWrapper(self.fw_rnn,
input_keep_prob=self.keep_prob)
self.bw_rnn = rnn_cell.GRUCell(self.hidden_size)
self.bw_rnn = DropoutWrapper(self.bw_rnn,
input_keep_prob=self.keep_prob)
def build_var(self):
with tf.variable_scope(self.name) as scope:
with tf.variable_scope('embedding'):
self.embedding = tf.get_variable('embedding',
initializer=tf.constant(
self.node_vec,
dtype=tf.float32))
with tf.variable_scope('BiGRU'):
self.gru_fw_cell = rnn_cell.GRUCell(self.n_hidden_gru)
self.gru_bw_cell = rnn_cell.GRUCell(self.n_hidden_gru)
with tf.variable_scope('attention'):
self.p_step = tf.get_variable('p_step',
initializer=self.initializer(
[1, self.n_steps]),
dtype=tf.float32)
self.a_geo = tf.get_variable('a_geo',
initializer=self.initializer([1]))
with tf.variable_scope('dense'):
self.weights = {
'dense1':
tf.get_variable('dense1_weight',
initializer=self.initializer([
2 * self.n_hidden_gru,
self.n_hidden_dense1
])),
'dense2':
tf.get_variable('dense2_weight',
initializer=self.initializer([
self.n_hidden_dense1,
self.n_hidden_dense2
])),
'out':
tf.get_variable('out_weight',
initializer=self.initializer(
[self.n_hidden_dense2, 1]))
}
self.biases = {
'dense1':
tf.get_variable('dense1_bias',
initializer=self.initializer(
[self.n_hidden_dense1])),
'dense2':
tf.get_variable('dense2_bias',
initializer=self.initializer(
[self.n_hidden_dense2])),
'out':
tf.get_variable('out_bias',
initializer=self.initializer([1]))
}
def testEmbeddingAttentionDecoder(self):
with self.test_session() as sess:
with variable_scope.variable_scope(
"root", initializer=init_ops.constant_initializer(0.5)):
inp = [constant_op.constant(0.5, shape=[2, 2])] * 2
cell_fn = lambda: rnn_cell.GRUCell(2)
cell = cell_fn()
enc_outputs, enc_state = rnn.static_rnn(cell, inp, dtype=dtypes.float32)
attn_states = array_ops.concat([
array_ops.reshape(e, [-1, 1, cell.output_size]) for e in enc_outputs
], 1)
dec_inp = [
constant_op.constant(
i, dtypes.int32, shape=[2]) for i in range(3)
]
# Use a new cell instance since the attention decoder uses a
# different variable scope.
dec, mem = seq2seq_lib.embedding_attention_decoder(
dec_inp,
enc_state,
attn_states,
cell_fn(),
num_symbols=4,
embedding_size=2,
output_size=3)
sess.run([variables.global_variables_initializer()])
res = sess.run(dec)
self.assertEqual(3, len(res))
self.assertEqual((2, 3), res[0].shape)
res = sess.run([mem])
self.assertEqual((2, 2), res[0].shape)
def test_timeseries_classification_sequential_tf_rnn(self):
np.random.seed(1337)
(x_train, y_train), _ = testing_utils.get_test_data(train_samples=100,
test_samples=0,
input_shape=(4,
10),
num_classes=2)
y_train = keras.utils.to_categorical(y_train)
model = keras.models.Sequential()
model.add(
keras.layers.RNN(rnn_cell.LSTMCell(5),
return_sequences=True,
input_shape=x_train.shape[1:]))
model.add(
keras.layers.RNN(
rnn_cell.GRUCell(y_train.shape[-1],
activation='softmax',
dtype=dtypes.float32)))
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizer_v2.adam.Adam(0.005),
metrics=['accuracy'],
run_eagerly=testing_utils.should_run_eagerly())
history = model.fit(x_train,
y_train,
epochs=15,
batch_size=10,
validation_data=(x_train, y_train),
verbose=2)
self.assertGreater(history.history['val_acc'][-1], 0.7)
_, val_acc = model.evaluate(x_train, y_train)
self.assertAlmostEqual(history.history['val_acc'][-1], val_acc)
predictions = model.predict(x_train)
self.assertEqual(predictions.shape, (x_train.shape[0], 2))
def testBlockGRUToGRUCellSingleStep(self):
with self.session(use_gpu=True, graph=ops.Graph()) as sess:
batch_size = 4
cell_size = 5
input_size = 6
seed = 1994
initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=seed)
# Inputs
x = array_ops.zeros([batch_size, input_size])
h = array_ops.zeros([batch_size, cell_size])
# Values for the inputs.
x_value = np.random.rand(batch_size, input_size)
h_value = np.random.rand(batch_size, cell_size)
# Output from the basic GRU cell implementation.
with vs.variable_scope("basic", initializer=initializer):
output = rnn_cell.GRUCell(cell_size)(x, h)
sess.run([variables.global_variables_initializer()])
basic_res = sess.run([output], {x: x_value, h: h_value})
# Output from the block GRU cell implementation.
with vs.variable_scope("block", initializer=initializer):
output = gru_ops.GRUBlockCell(cell_size)(x, h)
sess.run([variables.global_variables_initializer()])
block_res = sess.run([output], {x: x_value, h: h_value})
self.assertEqual(len(block_res), len(basic_res))
for block, basic in zip(block_res, basic_res):
self.assertAllClose(block, basic)
def model(self):
print('Building model\n')
# We don't want to modify to original tensor
x = self.x
# Reshape input into a list of tensors of the correct size
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, INPUT_SIZE])
# Since we're using one pixel at a time, transform list of vector of
# 784x1
x = tf.split(0, STEPS, x)
# Define LSTM cells and get outputs list and states
gru = rnn_cell.GRUCell(self.num_hid_units)
gru = rnn_cell.DropoutWrapper(gru, output_keep_prob=1)
if self.num_hid_layers > 1:
gru = rnn_cell.MultiRNNCell([gru] * self.num_hid_layers)
outputs, state = rnn.rnn(gru, x, dtype=tf.float32)
# Turn result back into [batch_size, steps, hidden_units] format.
outputs = tf.transpose(outputs, [1, 0, 2])
# Flatten into [batch_size x steps, hidden_units] to allow matrix
# multiplication
outputs = tf.reshape(outputs, [-1, self.num_hid_units])
# Apply affine transformation to reshape output [batch_size x steps, 1]
y1 = tf.matmul(outputs, self.weights_H2O) + self.bias_H2O
y1 = tf.reshape(y1, [-1, STEPS])
# Keep prediction (sigmoid applied) and non-sigmoid (apply sigmoid in
# cost function)
y_ns = y1[:, :783]
y_pred = tf.sigmoid(y1)[:, :783]
return y_ns, y_pred
def model(self):
cells = []
for i in range(1, len(self.layers_size) - 1):
if self.cell_type == 0:
cell = rnn_cell.BasicLSTMCell(self.layers_size[i])
elif self.cell_type == 1:
cell = rnn_cell.BasicRNNCell(self.layers_size[i])
elif self.cell_type == 2:
cell = rnn_cell.GRUCell(self.layers_size[i])
cell = rnn_cell.DropoutWrapper(cell,
output_keep_prob=self.keep_prob)
cells.append(cell)
multilayer_cell = rnn_cell.MultiRNNCell(cells)
multilayer_cell = rnn_cell.DropoutWrapper(
multilayer_cell, output_keep_prob=self.keep_prob)
output, state = tf.nn.dynamic_rnn(multilayer_cell,
self.input_tensor,
dtype=tf.float32)
output = tf.transpose(output, [1, 0, 2])
last = tf.gather(output,
int(output.get_shape()[0]) -
1) # This may be a bottleneck (memory)
last_weights = tf.Variable(
tf.random_normal([self.layers_size[-2], self.layers_size[-1]]))
if self.enable_bias:
bias = tf.Variable(tf.random_normal(([self.layers_size[-1]])))
return tf.nn.softmax(tf.matmul(last, last_weights) + bias)
return tf.nn.softmax(tf.matmul(last, last_weights))
def test_temporal_classification_sequential_tf_rnn(self):
with self.cached_session():
np.random.seed(1337)
(x_train,
y_train), _ = testing_utils.get_test_data(train_samples=100,
test_samples=0,
input_shape=(4, 10),
num_classes=2)
y_train = keras.utils.to_categorical(y_train)
model = keras.models.Sequential()
model.add(
keras.layers.RNN(rnn_cell.LSTMCell(5),
return_sequences=True,
input_shape=x_train.shape[1:]))
model.add(
keras.layers.RNN(
rnn_cell.GRUCell(y_train.shape[-1],
activation='softmax',
dtype=dtypes.float32)))
model.compile(loss='categorical_crossentropy',
optimizer=keras.optimizers.Adam(lr=0.1),
metrics=['accuracy'])
history = model.fit(x_train,
y_train,
epochs=15,
batch_size=16,
validation_data=(x_train, y_train),
verbose=2)
self.assertGreater(history.history['val_acc'][-1], 0.7)
def build_model(self):
target_v = tf.placeholder(tf.float32, [None])
conversation_d = tf.placeholder(tf.int32, [None, None])
conversation_g = tf.placeholder(tf.int32, [None, None])
conversation_len = tf.placeholder(tf.int32, [None])
last_description = tf.placeholder(tf.int32, [None])
target_word = tf.placeholder(tf.int32, [None])
i_know = tf.placeholder(tf.int32, [None])
word_embeddings = tf.get_variable('word_embed',
initializer=tf.convert_to_tensor(
self.data_loader.embeddings,
dtype=tf.float32),
trainable=False)
conv_d = tf.nn.embedding_lookup(word_embeddings, conversation_d)
conv_g = tf.nn.embedding_lookup(word_embeddings, conversation_g)
know = tf.nn.embedding_lookup(word_embeddings, i_know)
conv = tf.concat(2, [conv_g, conv_d])
with tf.variable_scope('guesser'):
last_des = tf.nn.embedding_lookup(word_embeddings,
last_description)
gue_cell = rnn_cell.GRUCell(self.hidden_units)
_, gue_state = rnn.dynamic_rnn(gue_cell,
conv,
conversation_len,
dtype=tf.float32)
gue_repr = tf.tanh(
rnn_cell._linear([gue_state, last_des], self.final_units,
True))
gue_core = tf.get_variable('gue_core',
[self.final_units, self.embedding_size])
gue_ready = tf.matmul(gue_repr, gue_core)
guesser_value = tf.reduce_sum(tf.mul(gue_ready, know), 1)
gue_pred = tf.matmul(gue_ready, word_embeddings, transpose_b=True)
_guess_ = tf.nn.top_k(gue_pred, self.vocab_size)
with tf.variable_scope('describer'):
target = tf.nn.embedding_lookup(word_embeddings, target_word)
des_cell = Contextual_GRUCell(self.hidden_units)
_, des_state = contextual_rnn(des_cell,
conv,
target,
conversation_len,
dtype=tf.float32)
des_repr = tf.tanh(
rnn_cell._linear([des_state, target], self.final_units, True))
des_core = tf.get_variable('des_core',
[self.final_units, self.embedding_size])
des_ready = tf.matmul(des_repr, des_core)
describer_value = tf.reduce_sum(tf.mul(des_ready, know), 1)
des_pred = tf.matmul(des_ready, word_embeddings, transpose_b=True)
_description_ = tf.nn.top_k(des_pred, self.vocab_size)
optimizer = tf.train.GradientDescentOptimizer(self.step_size)
update_guesser_op = optimizer.minimize(
tf.reduce_sum(tf.square(target_v - guesser_value)))
update_describler_op = optimizer.minimize(
tf.reduce_sum(tf.square(target_v - describer_value)))
return update_guesser_op, update_describler_op, guesser_value, describer_value, target_v, i_know, conversation_d, conversation_g, conversation_len, last_description, target_word, _guess_, _description_
def model(self):
"""
Builds the Tensorflow graph
:return:
"""
print('Building model\n')
# We don't want to modify to original tensor
x = self.x
# Reshape input into a list of tensors of the correct size
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, INPUT_SIZE])
# Since we're using one pixel at a time, transform list of vector of
# 784x1
x = tf.split(0, STEPS, x)
# Define LSTM cells and get outputs list and states
gru = rnn_cell.GRUCell(self.num_hid_units)
gru = rnn_cell.DropoutWrapper(gru, output_keep_prob=1)
gru = rnn_cell.MultiRNNCell([gru] * self.num_hid_layers)
outputs, state = rnn.rnn(gru, x, dtype=tf.float32)
# First affine-transformation - output from last input
y1 = tf.matmul(outputs[-1], self.weights_H2L) + self.bias_H2L
y2 = tf.nn.relu(y1)
y_pred = tf.matmul(y2, self.weights_L2O) + self.bias_L2O
return y_pred
def __init__(self, num_units, tied=False, non_recurrent_fn=None):
super(Grid2GRUCell, self).__init__(
num_units=num_units, num_dims=2,
input_dims=0, output_dims=0, priority_dims=0, tied=tied,
non_recurrent_dims=None if non_recurrent_fn is None else 0,
cell_fn=lambda n, i: rnn_cell.GRUCell(num_units=n, input_size=i),
non_recurrent_fn=non_recurrent_fn)
def RNN(x, init_state):
# Prepare data shape to match `rnn` function requirements
# Current data input shape: (batch_size, n_steps, n_hidden)
# Required shape: 'n_steps' tensors list of shape (batch_size, n_hidden)
# Permuting batch_size and n_steps
x = tf.transpose(x, [1, 0, 2])
# Reshaping to (n_steps*batch_size, n_hidden)
x = tf.reshape(x, [-1, n_hidden])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_hidden)
x = tf.split(0, seq_len, x)
# Define a lstm cell with tensorflow
#lstm_cell = rnn_cell.GRUCell(n_hidden, forget_bias=1.0)
gru_cell = rnn_cell.GRUCell(n_hidden)
# Get lstm cell output
outputs, states = rnn.rnn(gru_cell,
x,
dtype=tf.float32,
initial_state=init_state)
# Linear activation, using rnn inner loop last output
return outputs
def GRURNN(x, weights, biases):
x = tf.transpose(x, [1, 0, 2])
x = tf.reshape(x, [-1, nInput])
x = tf.split(x, nSteps, 0)
lstmCell = rnn_cell.GRUCell(nHidden)
outputs, states = rnn.static_rnn(lstmCell, x, dtype=tf.float32)
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def RNN(x, weights, biases, type, layer_norm):
# Prepare data shape to match `rnn` function requirements
# Current data input shape: (batch_size, n_steps, n_input)
# Required shape: 'n_steps' tensors list of shape (batch_size, n_input)
# Permuting batch_size and n_steps
x = tf.transpose(x, [1, 0, 2])
# Reshaping to (n_steps*batch_size, n_input)
x = tf.reshape(x, [-1, n_input])
# Split to get a list of 'n_steps' tensors of shape (batch_size, n_input)
x = tf.split(0, n_steps, x)
# Define a lstm cell with tensorflow
cell_class_map = {
"LSTM": rnn_cell.BasicLSTMCell(n_hidden),
"GRU": rnn_cell.GRUCell(n_hidden),
"BasicRNN": rnn_cell.BasicRNNCell(n_hidden),
"LNGRU": LNGRUCell(n_hidden),
"LNLSTM": LNBasicLSTMCell(n_hidden),
'HyperLnLSTMCell':HyperLnLSTMCell(n_hidden, is_layer_norm = layer_norm)}
lstm_cell = cell_class_map.get(type)
cell = rnn_cell.MultiRNNCell([lstm_cell] * FLAGS.layers)
print "Using %s model" % type
# Get lstm cell output
outputs, states = rnn.rnn(cell, x, dtype=tf.float32)
# Linear activation, using rnn inner loop last output
return tf.matmul(outputs[-1], weights['out']) + biases['out']
def setup_decoder(self):
if self.num_layers > 1:
self.decoder_cell = rnn_cell.GRUCell(self.size)
self.attn_cell = GRUCellAttn(self.size,
self.encoder_output,
scope="DecoderAttnCell")
with vs.variable_scope("Decoder"):
inp = self.decoder_inputs
for i in xrange(self.num_layers - 1):
with vs.variable_scope("DecoderCell%d" % i) as scope:
out, state_output = rnn.dynamic_rnn(
self.decoder_cell,
inp,
time_major=True,
dtype=dtypes.float32,
sequence_length=self.target_length,
scope=scope,
initial_state=self.decoder_state_input[i])
inp = self.dropout(out)
self.decoder_state_output.append(state_output)
with vs.variable_scope("DecoderAttnCell") as scope:
out, state_output = rnn.dynamic_rnn(
self.attn_cell,
inp,
time_major=True,
dtype=dtypes.float32,
sequence_length=self.target_length,
scope=scope,
initial_state=self.decoder_state_input[i + 1])
self.decoder_output = self.dropout(out)
self.decoder_state_output.append(state_output)
def get_episode(self, facts, question, memory):
'''The inner RNN of Episode Memory Networks.
We generate the episode using the final hidden state.
Arg:
facts: The output from Input Module shape of (N,F,H)
question: The output from Question Module shape of (N,H)
memory : The memory to be updated shape of (N,H)
'''
gru = rnn_cell.GRUCell(self.H)
c = tf.unpack(tf.transpose(facts, [1, 0, 2]),
self.F) # F-len List of Tensor shape of (N,H)
initial_state = tf.zeros([self.N, self.H])
hidden_state = initial_state
with tf.variable_scope('Episode') as scope:
for ct in c: # ct shape of (N,H)
ct.set_shape([self.N, self.H])
gt = self.attention_machanism(ct, question, memory)
hidden_state = gt * gru(
ct, hidden_state)[0] + (1 - gt) * hidden_state
scope.reuse_variables()
episode = hidden_state
return episode
def build_model(self):
'''
build model
'''
self._x = tf.placeholder(tf.int32, [self.batch_size], name='input')
self._y = tf.placeholder(tf.int32, [self.batch_size], name='output')
self.state = [tf.placeholder(tf.float32, [self.batch_size, \
self.rnn_size], name='rnn_state') for _ in range(self.layers)]
self.global_step = tf.Variable(0, name='global_step', trainable=False)
with tf.variable_scope('gru_layer'):
sigma = self.sigma if self.sigma != 0 else np.sqrt(6.0 / (self.n_items + self.rnn_size))
if self.init_as_normal:
initializer = tf.random_normal_initializer(mean=0, stddev=sigma)
else:
initializer = tf.random_uniform_initializer(minval=-sigma, maxval=sigma)
embedding = tf.get_variable('embedding', [self.n_items, \
self.rnn_size], initializer=initializer)
softmax_w = tf.get_variable('softmax_w', [self.n_items, \
self.rnn_size], initializer=initializer)
softmax_b = tf.get_variable('softmax_b', [self.n_items], \
initializer=tf.constant_initializer(0.0))
cell = rnn_cell.GRUCell(self.rnn_size, activation=self.hidden_act)
drop_cell = rnn_cell.DropoutWrapper(cell, output_keep_prob=self.dropout_p_hidden)
stacked_cell = rnn_cell.MultiRNNCell([drop_cell] * self.layers)
inputs = tf.nn.embedding_lookup(embedding, self._x)
output, state = stacked_cell(inputs, tuple(self.state))
self.final_state = state
if self.is_training:
# Use other examples of the minibatch as negative samples.
sampled_w = tf.nn.embedding_lookup(softmax_w, self._y)
sampled_b = tf.nn.embedding_lookup(softmax_b, self._y)
logits = tf.matmul(output, sampled_w, transpose_b=True) + sampled_b
self.yhat = self.final_activation(logits)
self.cost = self.loss_function(self.yhat)
else:
logits = tf.matmul(output, softmax_w, transpose_b=True) + softmax_b
self.yhat = self.final_activation(logits)
if not self.is_training:
return
self._lr = tf.maximum(1e-5, tf.train.exponential_decay(\
self.learning_rate, self.global_step, self.decay_steps, \
self.decay, staircase=True))
#Try different optimizers.
#optimizer = tf.train.AdagradOptimizer(self._lr)
optimizer = tf.train.AdamOptimizer(self._lr)
#optimizer = tf.train.AdadeltaOptimizer(self._lr)
#optimizer = tf.train.RMSPropOptimizer(self._lr)
tvars = tf.trainable_variables()
gvs = optimizer.compute_gradients(self.cost, tvars)
if self.grad_cap > 0:
capped_gvs = [(tf.clip_by_norm(grad, self.grad_cap), var) for grad, var in gvs]
else:
capped_gvs = gvs
self.train_op = optimizer.apply_gradients(capped_gvs, global_step=self.global_step)
def translate_model(X, y):
byte_list = skflow.ops.one_hot_matrix(X, 256)
in_X, in_y, out_y = skflow.ops.seq2seq_inputs(
byte_list, y, MAX_DOCUMENT_LENGTH, MAX_DOCUMENT_LENGTH)
cell = rnn_cell.OutputProjectionWrapper(rnn_cell.GRUCell(HIDDEN_SIZE), 256)
decoding, _, sampling_decoding, _ = rnn_seq2seq(in_X, in_y, cell)
return skflow.ops.sequence_classifier(decoding, out_y, sampling_decoding)
