class RPolicy:
def __init__(self, nact, rnn_units=256):
cells = [GRUCell(rnn_units, kernel_initializer=orthogonal()) for _ in range(2)]
self.gru = MultiRNNCell(cells)
self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32)
self.obs_ph = tf.placeholder(dtype=tf.float32)
fc1 = dense(self.obs_ph, rnn_units, activation=elu, kernel_initializer=orthogonal(), name='fc1')
expand = tf.expand_dims(fc1, axis=0, name='expand')
rnn_out, self.state = dynamic_rnn(self.gru, expand, initial_state=self.state)
reshape = tf.reshape(rnn_out, shape=[-1, rnn_units], name='reshape')
self.logits = dense(reshape, nact, kernel_initializer=orthogonal(0.01), name='logits')
self.pi = tf.nn.softmax(self.logits, name='pi')
self.action = boltzmann(self.pi)
self.value = dense(self.logits, 1, kernel_initializer=orthogonal(), name='value')
def forward(self, obs, sess):
feed_dict = {self.obs_ph: obs}
action, value = sess.run([self.action, self.pi], feed_dict=feed_dict)
return action, value
def reset(self):
self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32)
class CRPolicy:
def __init__(self, h, w, c, nact, rnn_units=256, cnn_units=32):
cells = [GRUCell(rnn_units, kernel_initializer=orthogonal()) for _ in range(2)]
self.gru = MultiRNNCell(cells)
self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32)
self.obs_ph = tf.placeholder(dtype=tf.float32, shape=[None, h, w, c])
cv1 = conv2d(self.obs_ph, cnn_units, 3, strides=2, activation=elu, kernel_initializer=orthogonal(), name='cv1')
cv2 = conv2d(cv1, cnn_units, 3, strides=2, activation=elu, kernel_initializer=orthogonal(), name='cv2')
cv3 = conv2d(cv2, cnn_units, 3, strides=2, activation=elu, kernel_initializer=orthogonal(), name='cv3')
cv4 = conv2d(cv3, cnn_units, 3, strides=2, activation=elu, kernel_initializer=orthogonal(), name='cv4')
flat = flatten(cv4, name='flatten')
fc1 = dense(flat, rnn_units, activation=elu, kernel_initializer=orthogonal(), name='fc1')
expand = tf.expand_dims(fc1, axis=0, name='expand')
rnn_out, self.state = dynamic_rnn(self.gru, expand, initial_state=self.state)
reshape = tf.reshape(rnn_out, shape=[-1, rnn_units], name='reshape')
self.logits = dense(reshape, nact, kernel_initializer=orthogonal(0.01), name='logits')
self.pi = tf.nn.softmax(self.logits, name='pi')
self.action = boltzmann(self.pi)
value = dense(reshape, 1, kernel_initializer=orthogonal(), name='value')
self.value = tf.squeeze(value)
def forward(self, obs, sess):
feed_dict = {self.obs_ph: obs}
action, value = sess.run([self.action, self.value], feed_dict=feed_dict)
return action, value
def reset(self):
self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32)
def __init__(self, batch_size, seq_length, n_layers, rnn_size, vocab_size,
scope, **ignored_args):
self.batch_size = batch_size
self.seq_length = seq_length
self.rnn_size = rnn_size
self.vocab_size = vocab_size
self.n_layers = n_layers
self.scope = scope
self.grad_clip = 5.0
self.input_data = tf.placeholder(tf.int32,
(self.seq_length, self.batch_size))
self.target_data = tf.placeholder(tf.int32,
(self.seq_length, self.batch_size))
self.embedding = tf.get_variable("embedding",
(self.vocab_size, self.rnn_size))
embedded = tf.nn.embedding_lookup(self.embedding, self.input_data)
# embedded.shape = (self.seq_length, self.batch_size, self.rnn_size)
self.softmax_w = tf.get_variable("softmax_w",
(self.rnn_size, self.vocab_size))
self.softmax_b = tf.get_variable("softmax_b", (self.vocab_size, ))
self.learning_rate = tf.placeholder(tf.float32, ())
cell = MultiRNNCell(
[BasicLSTMCell(self.rnn_size) for _ in range(self.n_layers)])
state = self.init_state = cell.zero_state(batch_size=self.batch_size,
dtype=tf.float32)
logits = [] # .shape = (seq_length, batch_size, vocab_size)
with tf.variable_scope(self.scope):
for i in range(self.seq_length):
output, state = cell(
embedded[i],
state) # output.shape = (batch_size, rnn_size)
logits.append(output @ self.softmax_w + self.softmax_b)
tf.get_variable_scope().reuse_variables()
self.final_state = state
loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
labels=self.target_data, logits=logits)
self.cost = tf.reduce_mean(loss)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self.cost, tvars),
self.grad_clip)
self.train_op = tf.train.AdamOptimizer(
self.learning_rate).apply_gradients(zip(grads, tvars))
# sample model
self.sample_input_char = tf.placeholder(tf.int32)
embedded = tf.nn.embedding_lookup(
self.embedding, tf.reshape(self.sample_input_char, (1, )))
self.sample_init_state = cell.zero_state(batch_size=1,
dtype=tf.float32)
with tf.variable_scope(self.scope, reuse=True):
output, self.sample_final_state = cell(embedded,
self.sample_init_state)
logits = output @ self.softmax_w + self.softmax_b
self.sample_output_probs = tf.nn.softmax(logits[0])
def self_rnn(input, units=128, layer_num = 2, parallel_iterations=64, name='gru', reuse=False):
with tf.variable_scope(name_or_scope=name):
with tf.variable_scope('enc'):
encoder_rnn = MultiRNNCell([GRUCell(units) for _ in range(layer_num)])
with tf.variable_scope('dec'):
decoder_rnn = MultiRNNCell([ResidualWrapper(GRUCell(units)) for _ in range(layer_num)])
rnn_tot = input.shape[1]
batch = input.shape[0]
cond = lambda x, *_: tf.less(x, rnn_tot)
with tf.variable_scope('pre'):
cnt = tf.zeros((), dtype=tf.int32)
encoder_init_state = encoder_rnn.zero_state(batch, dtype=tf.float32)
decoder_init_state = decoder_rnn.zero_state(batch, dtype=tf.float32)
res_ta = tf.TensorArray(dtype=tf.float32, size=rnn_tot)
input_time_major = tf.transpose(input, (1, 0, 2))
def body(cnt, encoder_pre, decoder_pre, res_ta):
input = input_time_major[cnt]
with tf.variable_scope('enc'):
output_enc, new_enc_state = encoder_rnn(input, encoder_pre)
with tf.variable_scope('dec'):
output_dec, new_dec_state = decoder_rnn(output_enc, decoder_pre)
res_ta = res_ta.write(cnt, output_dec)
cnt = tf.add(cnt, 1)
return cnt, new_enc_state, new_dec_state, res_ta
res_cnt, encoder_res, decoder_res, final_res_ta = tf.while_loop(cond, body, loop_vars=[cnt, encoder_init_state, decoder_init_state, res_ta], parallel_iterations=parallel_iterations)
# final_res_ta = tf.stack(final_res_ta)
final_res = final_res_ta.stack()
return final_res
def _dynamic_birnn(self, x, seq_len, batch_size, max_seq_len):
cell_fw = MultiRNNCell([DropoutWrapper(GRUCell(cell_hidden))
for cell_hidden in self.cell_hidden])
cell_bw = MultiRNNCell([DropoutWrapper(GRUCell(cell_hidden)) for cell_hidden in self.cell_hidden])
init_state_fw = cell_fw.zero_state(batch_size, dtype=tf.float32)
init_state_bw = cell_bw.zero_state(batch_size, dtype=tf.float32)
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=x,
initial_state_fw=init_state_fw,
initial_state_bw=init_state_bw,
sequence_length=seq_len
)
outputs = (outputs[0] + outputs[1]) / 2
outputs = tf.reduce_sum(outputs, axis=1)
outputs = tf.divide(outputs, tf.cast(seq_len[:, None], tf.float32))
fc = tf.layers.dense(outputs, 1000)
fc = tf.nn.leaky_relu(fc, 0.2)
fc = tf.layers.dense(fc, self.n_class)
return fc
def build_decoder_cell(self, encoder_outputs, encoder_final_state,
hidden_size, cell_type, layer_size):
"""
构建解码器所有层
:param encoder_outputs:
:param encoder_state:
:param hidden_size:
:param cell_type:
:param layer_size:
:return:
"""
sequence_length = self.encoder_inputs_length
if self.mode == 'decode':
encoder_outputs = tf.contrib.seq2seq.tile_batch(
encoder_outputs, multiplier=self.beam_width)
encoder_final_state = tf.contrib.seq2seq.tile_batch(
encoder_final_state, multiplier=self.beam_width)
sequence_length = tf.contrib.seq2seq.tile_batch(
sequence_length, multiplier=self.beam_width)
if self.bidirection:
cell = MultiRNNCell([
self.one_cell(hidden_size * 2, cell_type)
for _ in range(layer_size)
])
else:
cell = MultiRNNCell([
self.one_cell(hidden_size, cell_type)
for _ in range(layer_size)
])
# 使用attention机制
self.attention_mechanism = BahdanauAttention(
num_units=self.hidden_size,
memory=encoder_outputs,
memory_sequence_length=sequence_length)
def cell_input_fn(inputs, attention):
mul = 2 if self.bidirection else 1
attn_projection = layers.Dense(self.hidden_size * mul,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')
return attn_projection(array_ops.concat([inputs, attention], -1))
cell = AttentionWrapper(cell=cell,
attention_mechanism=self.attention_mechanism,
attention_layer_size=self.hidden_size,
cell_input_fn=cell_input_fn,
name='Attention_Wrapper')
if self.mode == 'decode':
decoder_initial_state = cell.zero_state(
batch_size=self.batch_size * self.beam_width,
dtype=tf.float32).clone(cell_state=encoder_final_state)
else:
decoder_initial_state = cell.zero_state(
batch_size=self.batch_size,
dtype=tf.float32).clone(cell_state=encoder_final_state)
return cell, decoder_initial_state
class CRPolicy(tf.keras.Model):
def __init__(self, n_actions, rnn_units=256, conv_units=32):
super(CRPolicy, self).__init__()
cells = [
GRUCell(rnn_units, kernel_initializer=orthogonal())
for _ in range(2)
]
self.gru = MultiRNNCell(cells)
self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32)
self.cv1 = Conv2D(conv_units,
3,
strides=2,
activation='elu',
kernel_initializer=orthogonal())
self.cv2 = Conv2D(conv_units,
3,
strides=2,
activation='elu',
kernel_initializer=orthogonal())
self.cv3 = Conv2D(conv_units,
3,
strides=2,
activation='elu',
kernel_initializer=orthogonal())
self.cv4 = Conv2D(conv_units,
3,
strides=2,
activation='elu',
kernel_initializer=orthogonal())
self.flatten = Flatten()
self.fc1 = Dense(rnn_units, kernel_initializer=orthogonal())
self.pol = Dense(n_actions, kernel_initializer=orthogonal(0.01))
self.val = Dense(1, kernel_initializer=orthogonal())
def call(self, obs):
x = tf.constant(obs, dtype=tf.float32)
x = self.cv1(x)
x = self.cv2(x)
x = self.cv3(x)
x = self.cv4(x)
x = self.flatten(x)
x = self.fc1(x)
x = tf.expand_dims(x, axis=0)
x, self.state = dynamic_rnn(self.gru, x, initial_state=self.state)
x = tf.reshape(x, shape=[-1, 256])
return self.pol(x), self.val(x)
def reset(self):
self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32)
def _simple_lstm(self, sequences):
with tf.variable_scope("sequence_encoder"):
bn = batch_norm(self.batch_size)
in_cell = MultiRNNCell(
[
# tf.contrib.rnn.DropoutWrapper(
LSTMCell(self.input_size, state_is_tuple=True)
# output_keep_prob=self.keep_prob,
# state_keep_prob=self.keep_prob
# ) for _ in range(self.num_layers)
],
state_is_tuple=True)
# state = tf.random_normal((self.batch_size, self.input_size))
# initial_state = (LSTMStateTuple(state, state),) * self.num_layers
initial_state = in_cell.zero_state(self.batch_size, tf.float32)
# using length we select the last output per sequence which
# represents the sequence encoding
self.length = tf.placeholder(tf.int32,
shape=(self.batch_size, ),
name="lengths")
self.enc_outs, self.enc_state = tf.nn.dynamic_rnn(
in_cell,
inputs=bn(sequences),
initial_state=initial_state,
sequence_length=self.length,
dtype=tf.float32)
return tf.gather_nd(
self.enc_outs,
tf.stack([tf.range(self.batch_size), self.length - 1], axis=1))
def simple_embed_rnn(inputs,
batch_size,
num_units,
num_layers,
num_residual_layers,
num_classes,
sequence_length=None,
initializer=None,
dropout=0,
unit_type='lstm'):
if initializer is None:
initializer = get_initializer('uniform', init_weight=math.sqrt(3))
embeddings = tf.get_variable('embedding_simple_rnn',
[num_classes, num_units],
initializer=initializer,
dtype=tf.float32)
encoder_inputs = tf.nn.embedding_lookup(embeddings, inputs)
if num_layers > 1:
cells = []
for i in range(num_layers):
cells.append(single_rnn_cell(unit_type,
num_units,
dropout,
residual_connection=(i >= num_layers - num_residual_layers)))
cell = MultiRNNCell(cells)
else:
cell = single_rnn_cell(unit_type, num_units, dropout)
state = cell.zero_state(batch_size, tf.float32)
encoder_outputs, encoder_state = tf.nn.dynamic_rnn(cell,
encoder_inputs,
sequence_length=sequence_length,
initial_state=state)
return encoder_outputs, encoder_state
def _dynamic_rnn(self, x, seq_len, batch_size, max_seq_len):
cell = MultiRNNCell([GRUCell(cell_hidden) for cell_hidden in self.cell_hidden])
init_state = cell.zero_state(batch_size, dtype=tf.float32)
outputs, state = tf.nn.dynamic_rnn(
cell,
inputs=x,
initial_state=init_state,
sequence_length=seq_len
)
if not self.avg_output:
index = tf.range(0, batch_size) * max_seq_len + (seq_len - 1)
outputs = tf.reshape(outputs, [-1, self.cell_hidden[-1]])
outputs = tf.gather(outputs, index)
else:
# outputs = tf.Print(outputs, [tf.shape(outputs)], message="output shape")
outputs = tf.reduce_sum(outputs, axis=1)
outputs = tf.divide(outputs, tf.cast(seq_len[:, None], tf.float32))
# outputs = tf.Print(outputs, [tf.shape(outputs)], message="output shape")
fc = tf.layers.dense(outputs, 1000)
fc = tf.nn.leaky_relu(fc, 0.2)
fc = tf.layers.dense(fc, self.n_class)
return fc
class DecoderRNNV1(RNNCell):
def __init__(self, out_units, attention_cell: AttentionRNN,
trainable=True, name=None, **kwargs):
super(DecoderRNNV1, self).__init__(name=name, trainable=trainable, **kwargs)
self._cell = MultiRNNCell([
OutputProjectionWrapper(attention_cell, out_units),
ResidualWrapper(GRUCell(out_units)),
ResidualWrapper(GRUCell(out_units)),
], state_is_tuple=True)
@property
def state_size(self):
return self._cell.state_size
@property
def output_size(self):
return self._cell.output_size
def zero_state(self, batch_size, dtype):
return self._cell.zero_state(batch_size, dtype)
def compute_output_shape(self, input_shape):
return tf.TensorShape([input_shape[0], input_shape[1], self.output_size])
def call(self, inputs, state):
return self._cell(inputs, state)
class Policy:
"""
Our policy. Takes as input the current state as a list of
input/feedback indices and outputs the action distribution.
"""
def __init__(self):
self.guess_embedding = Embedding(config.max_guesses + 1,
config.guess_embedding_size)
self.feedback_embedding = Embedding(config.max_feedback + 1,
config.feedback_embedding_size)
self.lstm = MultiRNNCell([
LSTMCell(config.lstm_hidden_size),
LSTMCell(config.lstm_hidden_size)
])
self.dense = tf.layers.Dense(config.max_guesses)
@property
def variables(self):
"""Return all the trainable parameters"""
return [
*self.guess_embedding.variables,
*self.feedback_embedding.variables,
*self.lstm.variables,
*self.dense.variables
]
@property
def named_variables(self):
"""Method to ensure variables across different 'Policy'
instances are named consistently"""
return dict(zip(map(str, itertools.count()), self.variables))
def __call__(self, game_state, with_softmax=True):
"""
Do a forward pass to get the action distribution
"""
state = self.lstm.zero_state(1, tf.float32)
for guess, feedback in game_state:
guess_tensor = tf.reshape(tf.convert_to_tensor(guess), (1,))
feedback_tensor = tf.reshape(tf.convert_to_tensor(feedback), (1,))
guess_embedded = self.guess_embedding(guess_tensor)
feedback_embedded = self.feedback_embedding(feedback_tensor)
combined_embedded = tf.concat([guess_embedded,
feedback_embedded],
axis=-1)
# can I do multiple inputs to the LSTM instead of concatenating?
output, state = self.lstm(combined_embedded, state)
logits = self.dense(output)
if with_softmax:
return tf.nn.softmax(logits)
return logits
def build_rnn(self, in_layer, nodes, batch_size, num_layers=2, mode='RNN'): if mode.upper()=='RNN': cell = MultiRNNCell([BasicRNNCell(nodes) for _ in range(num_layers)]) elif mode.upper()=='LSTM': cell = MultiRNNCell([BasicLSTMCell(nodes) for _ in range(num_layers)]) initial_state = cell.zero_state(batch_size, tf.float32) outputs, state = tf.nn.dynamic_rnn(cell, in_layer, initial_state=initial_state) return initial_state, outputs, state
def setup_decoder_cell(self, config, keep_prob, use_beam_search,
init_state, attention_states, attention_lengths):
batch_size = get_state_shape(init_state)[0]
if use_beam_search:
attention_states = tile_batch(attention_states,
multiplier=self.beam_width)
init_state = nest.map_structure(
lambda s: tile_batch(s, self.beam_width), init_state)
attention_lengths = tile_batch(attention_lengths,
multiplier=self.beam_width)
batch_size = batch_size * self.beam_width
attention_size = shape(attention_states, -1)
attention = getattr(tf.contrib.seq2seq, config.attention_type)(
attention_size,
attention_states,
memory_sequence_length=attention_lengths)
def cell_input_fn(inputs, attention):
# define cell input function to keep input/output dimension same
if not config.use_attention_input_feeding:
return inputs
attn_project = tf.layers.Dense(config.hidden_size,
dtype=tf.float32,
name='attn_input_feeding',
activation=self.activation)
return attn_project(tf.concat([inputs, attention], axis=-1))
cells = _setup_decoder_cell(config, keep_prob)
if config.top_attention: # apply attention mechanism only on the top decoder layer
cells[-1] = AttentionWrapper(
cells[-1],
attention_mechanism=attention,
name="AttentionWrapper",
attention_layer_size=config.hidden_size,
alignment_history=use_beam_search,
initial_cell_state=init_state[-1],
cell_input_fn=cell_input_fn)
init_state = [state for state in init_state]
init_state[-1] = cells[-1].zero_state(batch_size=batch_size,
dtype=tf.float32)
init_state = tuple(init_state)
cells = MultiRNNCell(cells)
else:
cells = MultiRNNCell(cells)
cells = AttentionWrapper(cells,
attention_mechanism=attention,
name="AttentionWrapper",
attention_layer_size=config.hidden_size,
alignment_history=use_beam_search,
initial_cell_state=init_state,
cell_input_fn=cell_input_fn)
init_state = cells.zero_state(batch_size=batch_size, dtype=tf.float32) \
.clone(cell_state=init_state)
return cells, init_state
def _dynamic_birnn(self, x, seq_len, batch_size, max_seq_len):
cell_fw = MultiRNNCell([GRUCell(cell_hidden) for cell_hidden in self.cell_hidden])
cell_bw = MultiRNNCell([GRUCell(cell_hidden) for cell_hidden in self.cell_hidden])
init_state_fw = cell_fw.zero_state(batch_size, dtype=tf.float32)
init_state_bw = cell_bw.zero_state(batch_size, dtype=tf.float32)
outputs, states = tf.nn.bidirectional_dynamic_rnn(
cell_fw=cell_fw,
cell_bw=cell_bw,
inputs=x,
initial_state_fw=init_state_fw,
initial_state_bw=init_state_bw,
sequence_length=seq_len
)
# outputs = tf.concat(outputs, 2)
#
# if not self.avg_output:
# index = tf.range(0, batch_size) * max_seq_len + (seq_len - 1)
# outputs = tf.reshape(outputs, [-1, self.cell_hidden[-1] * 2])
# outputs = tf.gather(outputs, index)
# else:
# outputs = tf.reduce_sum(outputs, axis=1)
# outputs = tf.divide(outputs, tf.cast(seq_len[:, None], tf.float32))
outputs = (outputs[0] + outputs[1]) / 2
if not self.avg_output:
index = tf.range(0, batch_size) * max_seq_len + (seq_len - 1)
outputs = tf.reshape(outputs, [-1, self.cell_hidden[-1]])
outputs = tf.gather(outputs, index)
else:
outputs = tf.reduce_sum(outputs, axis=1)
outputs = tf.divide(outputs, tf.cast(seq_len[:, None], tf.float32))
fc = tf.layers.dense(outputs, 1000)
fc = tf.nn.leaky_relu(fc, 0.2)
fc = tf.layers.dense(fc, self.n_class)
return fc
def unidirectional(self, inputs, controller_config, memory_unit_config, analyse=False, reuse=False):
"""
Connects unidirectional controller and memory unit and performs scan over sequence
Args:
inputs: TF tensor, input sequence
controller_config: dict, configuration of the controller
memory_unit_config: dict, configuration of the memory unit
analyse: bool, do analysis
reuse: bool, reuse
Returns: TF tensor, output sequence; TF tensor, hidden states
"""
with tf.variable_scope("controller"):
controller_list = get_rnn_cell_list(controller_config, name='controller', reuse=reuse, seed=self.seed,
dtype=self.dtype)
if controller_config['connect'] == 'sparse':
memory_input_size = controller_list[-1].output_size
mu_cell = get_memory_unit(memory_input_size, memory_unit_config, 'memory_unit', analyse=analyse,
reuse=reuse)
cell = MultiRNNCell(controller_list + [mu_cell])
else:
controller_cell = HolisticMultiRNNCell(controller_list)
memory_input_size = controller_cell.output_size
mu_cell = get_memory_unit(memory_input_size, memory_unit_config, 'memory_unit', analyse=analyse,
reuse=reuse)
cell = MultiRNNCell([controller_cell, mu_cell])
batch_size = inputs.get_shape()[1].value
cell_init_states = cell.zero_state(batch_size, dtype=self.dtype)
output_init = tf.zeros([batch_size, cell.output_size], dtype=self.dtype)
if analyse:
output_init = (output_init, mu_cell.analyse_state(batch_size, dtype=self.dtype))
init_states = (output_init, cell_init_states)
def step(pre_states, inputs):
pre_rnn_output, pre_rnn_states = pre_states
if analyse:
pre_rnn_output = pre_rnn_output[0]
controller_inputs = tf.concat([inputs, pre_rnn_output], axis=-1)
rnn_output, rnn_states = cell(controller_inputs, pre_rnn_states)
return (rnn_output, rnn_states)
outputs, states = tf.scan(step, inputs, initializer=init_states, parallel_iterations=32)
return outputs, states
def sequence2class_lstm_inference_graph(embedding_matrix,
batch_size=25,
num_layers=2,
num_classes=2,
state_size=100):
input_sequence = tf.placeholder(tf.int32, [batch_size, None],
name='input_sequence')
labels = tf.placeholder(tf.int32, [batch_size, 2], name='labels')
embeddings = tf.constant(embedding_matrix)
embeddings = tf.stop_gradient(embeddings)
rnn_input = tf.nn.embedding_lookup(embeddings, input_sequence)
lstm = BasicLSTMCell
lstms_list = [
lstm(state_size, state_is_tuple=True)
for _ in range(0, num_layers - 1)
]
cell = MultiRNNCell(lstms_list, state_is_tuple=True)
initial_state = cell.zero_state(batch_size, tf.float32)
rnn_outputs, final_state = tf.nn.dynamic_rnn(cell,
rnn_input,
initial_state=initial_state)
with tf.variable_scope('softmax'):
W = tf.get_variable('W', [state_size, num_classes], dtype=tf.float32)
b = tf.get_variable('b', [num_classes],
initializer=tf.constant_initializer(0.0),
dtype=tf.float32)
logits = tf.matmul(rnn_outputs[:, -1, :], W) + b
predictions = tf.nn.softmax(logits)
compare_score_label = tf.equal(tf.argmax(predictions, axis=1),
tf.argmax(labels, axis=1))
accuracy = tf.reduce_mean(tf.cast(compare_score_label, tf.float32))
return dict(input_sequence=input_sequence,
labels=labels,
accuracy=accuracy,
initial_state=initial_state,
final_state=final_state,
predictions=predictions)
class DecoderRNNV2(RNNCell):
def __init__(self,
out_units,
attention_cell: AttentionRNN,
is_training,
zoneout_factor_cell=0.0,
zoneout_factor_output=0.0,
lstm_impl=LSTMImpl.LSTMCell,
trainable=True,
name=None,
dtype=None,
**kwargs):
super(DecoderRNNV2, self).__init__(name=name,
trainable=trainable,
**kwargs)
self._cell = MultiRNNCell([
attention_cell,
ZoneoutLSTMCell(out_units,
is_training,
zoneout_factor_cell,
zoneout_factor_output,
lstm_impl=lstm_impl,
dtype=dtype),
ZoneoutLSTMCell(out_units,
is_training,
zoneout_factor_cell,
zoneout_factor_output,
lstm_impl=lstm_impl,
dtype=dtype),
],
state_is_tuple=True)
@property
def state_size(self):
return self._cell.state_size
@property
def output_size(self):
return self._cell.output_size
def zero_state(self, batch_size, dtype):
return self._cell.zero_state(batch_size, dtype)
def compute_output_shape(self, input_shape):
return tf.TensorShape(
[input_shape[0], input_shape[1], self.output_size])
def call(self, inputs, state):
return self._cell(inputs, state)
class RPolicy(tf.keras.Model):
def __init__(self, n_actions, rnn_units=256):
super(RPolicy, self).__init__()
cells = [
GRUCell(rnn_units, kernel_initializer=orthogonal())
for _ in range(2)
]
self.gru = MultiRNNCell(cells)
self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32)
self.fc1 = Dense(rnn_units, activation='relu')
self.pol = Dense(n_actions)
self.val = Dense(1)
def call(self, obs):
x = tf.constant(obs, dtype=tf.float32)
x = self.fc1(x)
x = tf.expand_dims(x, axis=0)
x, self.state = dynamic_rnn(self.gru, x, initial_state=self.state)
x = tf.reshape(x, shape=[-1, 256])
return self.pol(x), self.val(x)
def reset(self):
self.state = self.gru.zero_state(batch_size=1, dtype=tf.float32)
def _init(self, sequence, targets, authors):
batch_size = tf.shape(sequence)[0]
sequence_lengths = tf.cast(tf.count_nonzero(sequence, axis=1), tf.int32)
embedding = tf.Variable(
tf.random_normal((self._vocab_size, self._embed_size)),
name='char_embedding'
)
context = tf.Variable(
tf.random_normal((self._author_size, self._ctx_size)),
name='ctx_embedding'
)
embedded_sequence = tf.nn.embedding_lookup(embedding, sequence)
embedded_authors = tf.nn.embedding_lookup(context, authors)
gpu = lambda x: '/gpu:{}'.format(x % self._num_gpu)
if self._training:
dropout = lambda x: DropoutWrapper(
x, 1.0-self._input_dropout, 1.0-self._output_dropout)
helper = TrainingHelper(embedded_sequence, sequence_lengths)
else:
dropout = lambda x: x
helper = SampleEmbeddingHelper(embedding, sequence[:,0], 2)
base = lambda x: ContextWrapper(self._cell(x), embedded_authors)
wrap = lambda i, cell: DeviceWrapper(dropout(cell), gpu(i))
cells = [wrap(i, base(self._cell_size)) for i in range(self._cell_num)]
cell = MultiRNNCell(cells)
init_state = cell.zero_state(batch_size, tf.float32)
dense = tf.layers.Dense(
self._vocab_size, self._activation, name='fully_connected'
)
decoder = BasicDecoder(cell, helper, init_state, dense)
output, _, _ = dynamic_decode(decoder, swap_memory=True)
logits = output.rnn_output
weights = tf.sequence_mask(sequence_lengths, dtype=tf.float32)
loss = tf.contrib.seq2seq.sequence_loss(
logits,
targets,
weights
)
out = output.sample_id
return targets, loss, out
def _build_rnn_graph(self):
input_ = tf.placeholder(tf.float32,
[None, self.n_steps, self.n_inputs],
name='input')
labels = tf.placeholder(tf.float32, [None, self.n_classes],
name='labels')
with tf.name_scope("LSTM") as scope:
def single_cell():
return DropoutWrapper(GRUCell(self.n_hidden),
output_keep_prob=self._config.keep_prob)
multi_layer_cell = MultiRNNCell(
[single_cell() for _ in range(self._config.num_layers)])
initial_state = multi_layer_cell.zero_state(
self._config.batch_size, tf.float32)
output, state = tf.nn.dynamic_rnn(multi_layer_cell,
input_,
dtype=tf.float32)
output_flattened = tf.reshape(output, [-1, self.n_hidden])
with tf.name_scope("softmax") as scope:
with tf.variable_scope("softmax_params"):
softmax_w = tf.get_variable("softmax_w",
[self.n_hidden, self.n_classes])
softmax_b = tf.get_variable("softmax_b", [self.n_classes])
logits = tf.nn.xw_plus_b(output_flattened, softmax_w, softmax_b)
loss = tf.nn.sigmoid_cross_entropy_with_logits(labels=labels,
logits=logits,
name="sigmoid")
# Minimize loss using Adam Optimizer
self.optimizer = tf.train.AdamOptimizer(
learning_rate=self.learning_rate).minimize(loss)
# Make predictions
correct_prediction = tf.equal(tf.round(tf.nn.sigmoid(logits)), labels)
# Accuracy of individual classes being correct
self.acc_of_class = tf.reduce_mean(
tf.cast(correct_prediction, tf.float32))
# Accuracy of entire label being correct
all_labels_true = tf.reduce_min(
tf.cast(correct_prediction, tf.float32), 1)
self.acc_of_label = tf.reduce_mean(all_labels_true)
def _build_rnn_graph_lstm(self, inputs, weights, config, is_training):
"""Build the inference graph using canonical LSTM cells."""
# Slightly better results can be obtained with forget gate biases
# initialized to 1 but the hyperparameters of the model would need to be
# different than reported in the paper.
self.size = config.hidden_size
cell0_w, cell0_b, cell1_w, cell1_b = weights
cell0 = BayesianLSTMCell(self.size,
cell0_w,
cell0_b,
reuse=not is_training)
cell1 = BayesianLSTMCell(self.size,
cell1_w,
cell1_b,
reuse=not is_training)
cell = MultiRNNCell([cell0, cell1], state_is_tuple=True)
# cell = tf.contrib.rnn.BasicLSTMCell(
# config.hidden_size, forget_bias=0.0, state_is_tuple=True,
# reuse=not is_training)
#
# cell = tf.contrib.rnn.MultiRNNCell(
# [cell for _ in range(config.num_layers)], state_is_tuple=True)
self._initial_state = cell.zero_state(config.batch_size, data_type())
state = self._initial_state
outputs = []
with tf.variable_scope("RNN"):
for time_step in range(self.num_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
(cell_output, state) = cell(inputs[:, time_step, :], state)
outputs.append(cell_output)
output = tf.concat(outputs, 1)
output = tf.reshape(output, [-1, config.hidden_size])
return output, state
class RPolicy(tf.keras.Model):
def __init__(self, n_actions):
super(RPolicy, self).__init__()
cells = [GRUCell(128, kernel_initializer=orthogonal(np.sqrt(2))) for _ in range(2)]
self.gru = MultiRNNCell(cells)
self.s0 = self.gru.zero_state(batch_size=1, dtype=tf.float32)
self.fc1 = Dense(128, activation='relu', kernel_initializer=orthogonal(np.sqrt(2)))
self.fc2 = Dense(100, activation='relu', kernel_initializer=orthogonal(np.sqrt(2)))
self.fc3 = Dense(100, activation='relu', kernel_initializer=orthogonal(np.sqrt(2)))
self.pol = Dense(n_actions, kernel_initializer=orthogonal(0.01))
self.val = Dense(1, kernel_initializer=orthogonal(np.sqrt(2)))
def call(self, obs, state):
x = tf.constant(obs, dtype=tf.float32)
x = self.fc1(x)
x = tf.expand_dims(x, axis=0)
x, state = dynamic_rnn(self.gru, x, initial_state=state)
x = tf.reshape(x, shape=[-1, 128])
pi = self.fc2(x)
v = self.fc3(x)
return self.pol(pi), self.val(v), state
def RNN(X, weights, biases):
# hidden layer for input to cell
########################################
X = tf.reshape(X, [-1, n_inputs])
X_in = tf.matmul(X, weights['in']) + biases['in']
X_in = tf.reshape(X_in, [-1, n_steps, n_hidden_units])
# multi rnn cell
##########################################
cell = MultiRNNCell(
[BasicLSTMCell(n_hidden_units) for _ in range(layer_num)])
init_state = cell.zero_state(batch_size, dtype=tf.float32)
outputs, final_state = tf.nn.dynamic_rnn(cell,
X_in,
initial_state=init_state,
time_major=False)
# hidden layer for output as the final results
#############################################
outputs = tf.unstack(tf.transpose(outputs, [1, 0, 2]))
results = tf.matmul(outputs[-1],
weights['out']) + biases['out'] # shape = (128, 10)
return results
def decode(helper, scope, reuse=None):
with tf.variable_scope(scope, reuse=reuse):
rnn_layers = []
for i in range(n_decoder_layers):
# Create GRUCell with dropout. Do not forget to set the reuse flag properly.
cell = tf.nn.rnn_cell.GRUCell(hidden_size, reuse=reuse)
cell = tf.nn.rnn_cell.DropoutWrapper(
cell, input_keep_prob=self.dropout_ph)
rnn_layers.append(cell)
decoder_cell = MultiRNNCell(rnn_layers)
# Create a projection wrapper
decoder_cell = OutputProjectionWrapper(decoder_cell,
vocab_size,
reuse=reuse)
# Create BasicDecoder, pass the defined cell, a helper, and initial state
# The initial state should be equal to the final state of the encoder!
initial_state = decoder_cell.zero_state(batch_size=batch_size,
dtype=tf.float32)
decoder = BasicDecoder(decoder_cell,
helper,
initial_state=initial_state)
# The first returning argument of dynamic_decode contains two fields:
# * rnn_output (predicted logits)
# * sample_id (predictions)
max_iters = tf.reduce_max(self.ground_truth_lengths)
# max_iters = max_iter
outputs, _, _ = dynamic_decode(decoder=decoder,
maximum_iterations=max_iters,
output_time_major=False,
impute_finished=True)
return outputs
def __init__(self, is_training, config, input_):
self._is_training = is_training
self._input = input_
self.batch_size = input_.batch_size
self.num_steps = input_.num_steps
self._input_data = input_.input_data
size = config.X_dim
hidden_size = config.hidden_size
vocab_size = config.vocab_size
self._targets = input_.targets
# Construct prior
prior = Prior(config.prior_pi, config.log_sigma1, config.log_sigma2)
# Fetch embeddings
inputs = input_.input_data
# Build the BBB LSTM cells
cells = []
for i in range(config.num_layers):
if (i == 0):
LSTM_input_size = config.X_dim
else:
LSTM_input_size = config.hidden_size
cells.append(BayesianLSTMCell(LSTM_input_size, config.hidden_size, prior, is_training,
forget_bias=0.0,
name="bbb_lstm_{}".format(i)))
cell = MultiRNNCell(cells, state_is_tuple=True)
self._initial_state = cell.zero_state(config.batch_size, data_type())
state = self._initial_state
# Forward pass for the truncated mini-batch
outputs = []
with tf.variable_scope("RNN"):
for time_step in range(self.num_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
(cell_output, state) = cell(inputs[:, time_step, :], state)
outputs.append(cell_output)
output = tf.reshape(tf.concat(outputs, 1), [-1, hidden_size])
# Softmax weights
softmax_w = sample_posterior((hidden_size, vocab_size), "softmax_w", prior, is_training)
softmax_b = sample_posterior((vocab_size, 1), "softmax_b", prior, is_training)
logits = tf.nn.xw_plus_b(output, softmax_w, tf.squeeze(softmax_b))
logits = tf.reshape(logits, [self.batch_size, self.num_steps, vocab_size])
self._output = tf.nn.softmax(logits)
loss = tf.contrib.seq2seq.sequence_loss(
logits,
input_.targets,
tf.ones([self.batch_size, self.num_steps], dtype=data_type()),
average_across_timesteps=False,
average_across_batch=False)
# Update the cost
# Remember to divide by batch size
self._cost = tf.reduce_sum(loss) / self.batch_size
self._kl_loss = 0.
self._final_state = state
if not is_training:
return
#Compute KL divergence
#B = number of batches aka the epoch size
#C = number of truncated sequences in a batch aka batch_size variable
B = self._input.epoch_size
C = self.batch_size
kl_loss = tf.add_n(tf.get_collection("KL_layers"), "kl_divergence")
kl_factor = 1.0/(B*C)
self._kl_loss = kl_factor * kl_loss
self._total_loss = self._cost + self._kl_loss
self._lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self._total_loss, tvars),
config.max_grad_norm)
optimizer = tf.train.GradientDescentOptimizer(self._lr)
self._train_op = optimizer.apply_gradients(
zip(grads, tvars),
global_step=tf.contrib.framework.get_or_create_global_step())
self._new_lr = tf.placeholder(data_type(), shape=[], name="new_learning_rate")
self._lr_update = tf.assign(self._lr, self._new_lr)
def __init__(self, data_size, time_len, unit_size, num_layers, batch_size,
learning_rate, feed_previous):
'''
Create the basic encoder-decoder seq2seq model
:param unit_size: number of units in each LSTM layer of the model
:param num_layers: number of LSTM layers in the model
:param batch_size: the size of batches used during training
:param learning_rate:
'''
self.input_size = data_size
self.time_len = time_len
self.batch_size = batch_size
self.learning_rate = tf.Variable(float(learning_rate),
trainable=False,
name='lr')
self.global_step = tf.Variable(0, trainable=False, name='global_step')
def single_cell():
return BasicLSTMCell(unit_size)
cell = single_cell()
if num_layers > 1:
cell = MultiRNNCell([single_cell() for _ in range(num_layers)])
print('state size', cell.state_size)
print('zero state size',
cell.zero_state(self.batch_size, dtype=tf.float32))
# Set placeholder for encoder's inputs
self.encoder_inputs = []
self.decoder_inputs = []
for i in range(self.time_len):
self.encoder_inputs.append(
tf.placeholder(shape=[self.batch_size, self.input_size],
name='encoder{}'.format(i),
dtype=tf.float32))
self.decoder_inputs.append(
tf.placeholder(shape=[self.batch_size, self.input_size],
name='decoder{}'.format(i),
dtype=tf.float32))
# The purpose is reconstruction, thus the targets should be the reverse of the input
targets = self.encoder_inputs[::-1]
outputs, _ = advanced_rnn_seq2seq(
encoder_inputs=self.encoder_inputs,
decoder_inputs=self.decoder_inputs,
cell=cell,
num_decoder_symbols=self.input_size,
output_projection=None,
feed_previous=feed_previous
) # the outputs have been projected based on the original lstm outputs
targets = tf.stack(targets, axis=1)
self.outputs = tf.stack(outputs, axis=1)
self.loss = tf.losses.mean_squared_error(targets, self.outputs)
self.error_vector = tf.abs(self.outputs - targets)
# set up the train operation
optimizer = tf.train.AdamOptimizer(learning_rate=self.learning_rate)
self.train_op = optimizer.minimize(self.loss,
global_step=self.global_step)
# the saver for handling all parameters for the model
self.saver = tf.train.Saver(tf.global_variables())
def __init__(self, is_training, config, input_):
"""
This initializer function will read the hyperparameters, from that it will
set the atchitecture of the network.
The is_training flag is nice to build the network. If it is not for training then we do not
need to builf to the graph the loss function and optimizer.
"""
# Variable to know if the model is being used for training
self._is_training = is_training
# TODO: This is the structure we just saw...
self._input = input_
# Setting the chains properties
self.batch_size = config.batch_size
self.num_steps = input_.num_steps
self._input_data = input_.input_data
input_data_ids = input_.input_data
self._targets = input_.targets
# Setting the architectute properties
# Dimensionality of the input !!
# TODO: For now we set it the same as the hidden_size. Probably for matrix concatenation purposes ?
# Dimensionality of the output ! In the case of classification, the cardinality of the output
Y_cardinality = config.Y_cardinality # Size of the output
# Construct prior
prior = VI.Prior(config.prior_pi, config.log_sigma1, config.log_sigma2)
########################################################################
############# Transform Categorial values (words) into real values vectors ############
########################################################################
# Fetch embeddings
# with tf.device("/cpu:0"):
# embedding = VI.sample_posterior([vocab_size, size], "embedding", prior, is_training)
# inputs = tf.nn.embedding_lookup(embedding, input_.input_data)
# If we have discrete input X and we want to embed them in random vectors of size "size"
# We also need to include the cardinality of the output Y.
# if (type(config.X_dim) != type(None)):
if (config.embedding == True):
with tf.device("/cpu:0"):
embedding = tf.get_variable(
"embedding", [Y_cardinality, config.X_dim], dtype=VI.data_type())
inputs = tf.nn.embedding_lookup(embedding, input_data_ids)
X_dim = config.X_dim
else:
X_dim = config.X_dim# inputs.get_shape()[-1].value
# inputs = tf.get_variable("Continous_data_input", [self.batch_size,self.num_steps, X_dim], dtype=VI.data_type(), trainable = False)
# inputs.assign(input_data_ids)
#
# caca = tf.zeros_initializer(tf.int32)((self.batch_size,Y_cardinality, tf.int32))
# targets = tf.get_variable("Discrete_Target", [self.batch_size,Y_cardinality], dtype=tf.int32, trainable = False,
# initializer = caca)
# targets.assign(input_.targets)
# inputs = tf.Variable(input_data_ids, trainable = False)
# targets = tf.Variable(input_.targets, trainable = False)
inputs = input_data_ids
targets = input_.targets
# These are the chains in the Batch. They are represented by a 3D tensor with dimensions
# - size_epoch: Number of chains in the batch
# - num_steps: Number of elements of the chain
# - D: Dimensionality of the elements of the chain.
# TODO: maybe due to the initial embedding that has to be done, all inputs are given when defining the model,
# we do not want that, we want them to be in a way where do the preprocessing before and we have chains as placeholder.
input_chains = inputs[:, :, :]
print ("-----------------------------")
print ("Input Batch X shape", inputs.shape)
print ("Input Batch Y shape", targets.shape)
print ("Input_size: %i"%X_dim)
print ("Output_size: %i"%Y_cardinality)
print ("Number of chains in a batch: %i"%self.batch_size)
print ("Number of elements in a chain: %i"%self.num_steps)
print ("Number of hidden state neurons LTSM: %i"%config.hidden_size)
########################################################################
############# Start Building the Architecute of the Network ############
########################################################################
######################################################################
################ Build and Stack BBB LSTM cells ################
cells = []
for i in range(config.num_layers):
if (i == 0):
LSTM_input_size = X_dim
else:
LSTM_input_size = config.hidden_size
cells.append(BLC.BayesianLSTMCell(LSTM_input_size, config.hidden_size, prior, is_training,
forget_bias=0.0,
name="bbb_lstm_{}".format(i)))
# The following line will stack the LSTM cells together
# They just need to follow the interface that we already wrote
# Notice we use state_is_tuple=True since the LSTM cells have 2 states C_t and h_t
DeepLSTMRNN = MultiRNNCell(cells, state_is_tuple=True)
# Initialize the state values to 0 ?
# TODO: We need to provide info about the Batch size ? That is the number of chains
# we want to compute the output at once.
#####################################################################################
################ Propagate the chains in the batch from input to output ################
# Initialization.
# This is the initial state for the LSTM when we feed it a new chain (is it just the 0s) probably. Then it should output the conditional most lilkely word.
# We need to give it the batch_size because we are going to propagate the chains in parallel.
# initial state will have dimensions [batch_size, (LSTM_hidden_size, LSTM_hidden_size)] since each state of the LSTM is made of the previous
self._initial_state = DeepLSTMRNN.zero_state(config.batch_size, VI.data_type())
state = self._initial_state
# Forward pass for the truncated mini-batch
# hs_o: This list will contain in each of its elements,
# the hidden state of the last LSTM of the network
# for each of the number of steps (length of the chains that is has to be the same for every chain).
# Each of this hidden states has dimensions [LSTM_hidden_size, num_batch] since we are computing in parallel for all chains in the batch.
# Now we propagate the chains in parallel and the initial state through the Deep Bayesian LSTM.
# At each time step we will save the hidden state of the last LSTM to convert it later to the real output and being able
# to compute the cost function and the output !
# TODO: This is probably why we want the chains to have the same length. Also maybe to not having to worry later to weight the
# cost functions by the length of the chains. Anyway... for now we will just accept it.
hs_o = []
with tf.variable_scope("RNN"): # We put all the LSTMs under the name RNN.
for time_step in range(self.num_steps): # For each element in the chain
if (time_step > 0): # Maybe this is so that we do not create the LSTMS a lot of times in the TensorBoard ?
tf.get_variable_scope().reuse_variables()
# Now we start feeding the time_step-th element of each of the chains at the same time to the network, obtaining the state for
(cell_output, state) = DeepLSTMRNN(input_chains[:,time_step,:], state)
hs_o.append(cell_output)
print (["size output state LSTM", cell_output.shape])
# print ("Num steps: %i"%self.num_steps)
# Now we concatenate all the hidden spaces of dimension [num_batch, LSTM_hidden_size]
# into in the list with dimension [num_batch x step_size, LSTM_hidden_size]. At the end of the day
# all of the hidden spaces will be multiplied by the same weights of the dense softmax layer so we concatenate all of the
# output hidden spaces for later multiplication.
hs_o = tf.reshape( tf.concat(hs_o, 1), [-1, config.hidden_size])
print (["Size of the Concatenated output state of the last LSTM for all chains in batch and time-steps in a batch", hs_o.shape])
######################################################################
################ Build the output layer ############################
# In our case the output later is just a dense layer that transforms the hidden space
# of the last LSTM into the prediction of each discrete output (word), applying a softmax
# function to the output of the neurons.
# The parameters of this layer are just the Weights and biases of it.
# The next call function will create the weights if they have not been create before.
# Identified by the names ""
# TODO: Not really a TODO, but the important part here is that we changed size vy config.hidden_size
softmax_w = VI.sample_posterior((config.hidden_size , Y_cardinality), "softmax_w", prior, is_training)
softmax_b = VI.sample_posterior((Y_cardinality, 1), "softmax_b", prior, is_training)
print ("Shape of the weights of the output Dense layer",softmax_w.shape)
print ("Shape of the weights of the output Dense layer",softmax_b.shape)
## We propagate the hidden spaces through the network in order to obtain the outout of the network before
## the softmax function, which is called the logits. This logits will have dimensions
## [num_batch x step_size, LSTM_hidden_size] that we need to break down further.
# Logits are the input to the softmax layer !
logits = tf.nn.xw_plus_b(hs_o, softmax_w, tf.squeeze(softmax_b))
# We reshape it back to the proper form [chain, sample, output]
print ("Shape of logits after multiplication of ohs", logits.shape)
logits = tf.reshape(logits, [self.batch_size, self.num_steps, Y_cardinality])
print ("Shape of logits after reshpaing", logits.shape)
# We can compute the output of the chains !
# TODO: maybe do not execute this line in the training model to save computation ? Maybe it wouldnt be executed anyway ?
self._output = tf.nn.softmax(logits)
""" This is finally the output of the batch, our prediction of the word,
for each of the words in the batch. Since we have:
- self.batch_size number of chains in the batch
- Each chain has the same number of words: self.num_steps
- The prediction of each word is the probability of each of the vocab_size variables
"""
#####################################################################################
################ Setting the Loss function ################
#####################################################################################
#B = number of batches aka the epoch size
#C = number of truncated sequences in a batch aka batch_size variable
B = self._input.epoch_size
C = self.batch_size
loss = tf.contrib.seq2seq.sequence_loss(
logits,
targets,
tf.ones([self.batch_size, self.num_steps], dtype=VI.data_type()),
average_across_timesteps=False,
average_across_batch=False)
# Update the cost
# Remember to divide by batch size
self._cost = tf.reduce_sum(loss) / self.batch_size
self._kl_loss = 0.
self._final_state = state
if not is_training:
return
#Compute KL divergence
## We get the KL loss that was computed during the sampling of the variational posterior !!
kl_loss = tf.add_n(tf.get_collection("KL_layers"), "kl_divergence")
self._kl_loss = kl_loss /(B*C)
# Compute the final loss, this is a proportion between the likelihood of the data (_cost)
# And the KL divergence of the posterior
# TODO: Remove increased by 2 the cost so that the total cost is more influenced
# on the data !
self._total_loss = self._cost + self._kl_loss
#####################################################################################
################ Setting the training algorithm ################
#####################################################################################
## Set the trainable variables, the variables for which the gradient with respect to the loss function
# will be computed and will be modified by the optimizer when the session is run :)
self._lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(tf.gradients(self._total_loss, tvars),
config.max_grad_norm)
optimizer = tf.train.GradientDescentOptimizer(self._lr)
self._train_op = optimizer.apply_gradients(
zip(grads, tvars),
global_step=tf.contrib.framework.get_or_create_global_step())
self._new_lr = tf.placeholder(VI.data_type(), shape=[], name="new_learning_rate")
self._lr_update = tf.assign(self._lr, self._new_lr)
def __init__(self,
inp,
inp_mask,
decode_time_steps,
hyper_params=None,
name='Tacotron'):
"""
Build the computational graph.
:param inp:
:param inp_mask:
:param decode_time_steps:
:param hyper_params:
:param name:
"""
super(Tacotron, self).__init__(name)
self.hyper_params = HyperParams(
) if hyper_params is None else hyper_params
with tf.variable_scope(name):
self.global_step = tf.Variable(0,
name='global_step',
trainable=False)
batch_size = tf.shape(inp)[0]
input_time_steps = tf.shape(inp)[1]
reduc = self.hyper_params.reduction_rate
output_time_steps = decode_time_steps * reduc
### Encoder [begin]
with tf.variable_scope('character_embedding'):
embed_inp = EmbeddingLayer(self.hyper_params.embed_class,
self.hyper_params.embed_dim)(inp)
with tf.variable_scope('encoder_pre_net'):
pre_ed_inp = tf.layers.dropout(tf.layers.dense(
embed_inp, 256, tf.nn.relu),
training=False)
pre_ed_inp = tf.layers.dropout(tf.layers.dense(
pre_ed_inp, 128, tf.nn.relu),
training=False)
encoder_output = modules.cbhg(pre_ed_inp,
training=False,
k=16,
bank_filters=128,
projection_filters=(128, 128),
highway_layers=4,
highway_units=128,
bi_gru_units=128,
sequence_length=inp_mask,
name='encoder_cbhg',
reuse=False)
### Encoder [end]
### Attention Module
with tf.variable_scope('attention'):
att_module = AttentionModule(256,
encoder_output,
sequence_length=inp_mask,
time_major=False)
### Decoder [begin]
att_cell = ZoneoutWrapper(sGRUCell(256), 0.1, False)
dec_cell = MultiRNNCell(
[ResidualWrapper(GRUCell(256)) for _ in range(2)])
# prepare output alpha TensorArray
with tf.variable_scope('prepare_decode'):
# prepare output alpha TensorArray
reduced_time_steps = tf.div(output_time_steps, reduc)
init_att_cell_state = att_cell.zero_state(
batch_size, tf.float32)
init_dec_cell_state = dec_cell.zero_state(
batch_size, tf.float32)
init_state_tup = tuple(
[init_att_cell_state, init_dec_cell_state])
init_output_ta = tf.TensorArray(size=reduced_time_steps,
dtype=tf.float32)
init_alpha_ta = tf.TensorArray(size=reduced_time_steps,
dtype=tf.float32)
go_array = tf.zeros(
[batch_size, self.hyper_params.seq2seq_dim],
dtype=tf.float32)
init_context = tf.zeros([batch_size, 256], dtype=tf.float32)
init_time = tf.constant(0, dtype=tf.int32)
cond = lambda x, *_: tf.less(x, reduced_time_steps)
def body(this_time, old_output_ta, old_alpha_ta, old_state_tup,
last_context, last_output):
with tf.variable_scope('decoder_pre_net'):
dec_pre_ed_inp = last_output
dec_pre_ed_inp = tf.layers.dropout(tf.layers.dense(
dec_pre_ed_inp, 256, tf.nn.relu),
training=True)
dec_pre_ed_inp = tf.layers.dropout(tf.layers.dense(
dec_pre_ed_inp, 128, tf.nn.relu),
training=True)
with tf.variable_scope('attention_rnn'):
att_cell_inp = tf.concat([last_context, dec_pre_ed_inp],
axis=-1)
att_cell_out, att_cell_state = att_cell(
att_cell_inp, old_state_tup[0])
with tf.variable_scope('attention'):
query = att_cell_state
context, alpha = att_module(query)
new_alpha_ta = old_alpha_ta.write(this_time, alpha)
with tf.variable_scope('decoder_rnn'):
dec_input = tf.layers.dense(
tf.concat([att_cell_out, context], axis=-1), 256)
dec_cell_out, dec_cell_state = dec_cell(
dec_input, old_state_tup[1])
dense_out = tf.layers.dense(
dec_cell_out, self.hyper_params.seq2seq_dim * reduc)
new_output_ta = old_output_ta.write(this_time, dense_out)
new_output = dense_out[:, -self.hyper_params.seq2seq_dim:]
new_state_tup = tuple([att_cell_state, dec_cell_state])
return tf.add(
this_time, 1
), new_output_ta, new_alpha_ta, new_state_tup, context, new_output
# run loop
_, seq2seq_output_ta, alpha_ta, *_ = tf.while_loop(
cond, body, [
init_time, init_output_ta, init_alpha_ta, init_state_tup,
init_context, go_array
])
with tf.variable_scope('reshape_decode'):
seq2seq_output = tf.reshape(
seq2seq_output_ta.stack(),
shape=(reduced_time_steps, batch_size,
self.hyper_params.seq2seq_dim * reduc))
seq2seq_output = tf.reshape(
tf.transpose(seq2seq_output, perm=(1, 0, 2)),
shape=(batch_size, output_time_steps,
self.hyper_params.seq2seq_dim))
self.seq2seq_output = seq2seq_output
alpha_output = tf.reshape(alpha_ta.stack(),
shape=(reduced_time_steps,
batch_size, input_time_steps))
alpha_output = tf.expand_dims(
tf.transpose(alpha_output, perm=(1, 0, 2)), -1)
self.alpha_output = alpha_output
### Decoder [end]
### PostNet [begin]
post_output = modules.cbhg(
seq2seq_output,
training=False,
k=8,
bank_filters=128,
projection_filters=(256, self.hyper_params.seq2seq_dim),
highway_layers=4,
highway_units=128,
bi_gru_units=128,
sequence_length=None,
name='decoder_cbhg',
reuse=False)
post_output = tf.layers.dense(post_output,
self.hyper_params.post_dim,
name='post_linear_transform')
self.post_output = post_output
def __init__(self, inp, inp_mask, decode_time_steps, ctr_flag, ctr_attention, hyper_params=None, name='Tacotron'):
"""
Build the computational graph.
:param inp:
:param inp_mask:
:param decode_time_steps:
:param hyper_params:
:param name:
"""
super(Tacotron, self).__init__(name)
self.hyper_params = HyperParams() if hyper_params is None else hyper_params
with tf.variable_scope(name):
self.global_step = tf.Variable(0, name='global_step', trainable=False)
batch_size = tf.shape(inp)[0]
input_time_steps = tf.shape(inp)[1]
reduc = self.hyper_params.reduction_rate
output_time_steps = decode_time_steps * reduc
### Encoder [begin]
with tf.variable_scope('character_embedding'):
embed_inp = EmbeddingLayer(self.hyper_params.embed_class, self.hyper_params.embed_dim)(inp)
with tf.variable_scope("changeToVarible"):
self.single_style_token = tf.get_variable('style_token', (1, self.hyper_params.styles_kind, self.hyper_params.style_dim), dtype=tf.float32)
self.style_token = tf.tile(self.single_style_token, (batch_size, 1, 1))
with tf.variable_scope('encoder_pre_net'):
pre_ed_inp = tf.layers.dropout(tf.layers.dense(embed_inp, 256, tf.nn.relu), training=False)
pre_ed_inp = tf.layers.dropout(tf.layers.dense(pre_ed_inp, 128, tf.nn.relu), training=False)
encoder_output = modules.cbhg(pre_ed_inp, training=False, k=16, bank_filters=128,
projection_filters=(128, 128), highway_layers=4, highway_units=128,
bi_gru_units=128, sequence_length=inp_mask,
name='encoder_cbhg', reuse=False)
with tf.variable_scope('post_text'):
all_outputs, _ = tf.nn.dynamic_rnn(cell=GRUCell(256), inputs=encoder_output, sequence_length=inp_mask,
dtype=encoder_output.dtype, parallel_iterations=unkonwn_parallel_iterations)
all_outputs = tf.transpose(all_outputs, [1, 0, 2])
static_encoder_output = all_outputs[-1]
### Encoder [end]
### Attention Module
with tf.variable_scope('attention'):
att_module = AttentionModule(256, encoder_output, sequence_length=inp_mask, time_major=False)
with tf.variable_scope("attention_style"):
att_module_style = AttentionModule(256, self.style_token, time_major=False)
### Decoder [begin]
att_cell = GRUCell(256)
dec_cell = MultiRNNCell([ResidualWrapper(GRUCell(256)) for _ in range(2)])
# prepare output alpha TensorArray
with tf.variable_scope('prepare_decode'):
# prepare output alpha TensorArray
reduced_time_steps = tf.div(output_time_steps, reduc)
init_att_cell_state = att_cell.zero_state(batch_size, tf.float32)
init_dec_cell_state = dec_cell.zero_state(batch_size, tf.float32)
init_state_tup = tuple([init_att_cell_state, init_dec_cell_state])
init_output_ta = tf.TensorArray(size=reduced_time_steps, dtype=tf.float32)
init_alpha_ta = tf.TensorArray(size=reduced_time_steps, dtype=tf.float32)
init_weight_ta = tf.TensorArray(size=reduced_time_steps, dtype=tf.float32)
init_weight_per_ta = tf.TensorArray(size=reduced_time_steps, dtype=tf.float32)
init_alpha_style_ta = tf.TensorArray(size=reduced_time_steps, dtype=tf.float32)
go_array = tf.zeros([batch_size, self.hyper_params.seq2seq_dim], dtype=tf.float32)
init_context = tf.zeros([batch_size, 256], dtype=tf.float32)
init_context_style = tf.zeros([batch_size, 256], dtype=tf.float32)
init_time = tf.constant(0, dtype=tf.int32)
cond = lambda x, *_: tf.less(x, reduced_time_steps)
def body(this_time, old_output_ta, old_alpha_ta, old_alpha_style_ta, old_weight_ta, old_weight_per_ta,
old_state_tup, last_context, last_context_style, last_output):
with tf.variable_scope('decoder_pre_net'):
dec_pre_ed_inp = last_output
dec_pre_ed_inp = tf.layers.dropout(tf.layers.dense(dec_pre_ed_inp, 256, tf.nn.relu), training=False)
dec_pre_ed_inp = tf.layers.dropout(tf.layers.dense(dec_pre_ed_inp, 128, tf.nn.relu), training=False)
with tf.variable_scope('attention_rnn'):
# dec_pre_ed_inp = tf.Print(dec_pre_ed_inp, [dec_pre_ed_inp[0]], message='dec', summarize=10)
att_cell_inp = tf.concat([last_context, dec_pre_ed_inp], axis=-1)
att_cell_out, att_cell_state = att_cell(att_cell_inp, old_state_tup[0])
with tf.variable_scope('attention'):
query = att_cell_state[0]
context, alpha = att_module(query)
new_alpha_ta = old_alpha_ta.write(this_time, alpha)
with tf.variable_scope("attention_style"):
query_style = att_cell_state[0]
context_style, alpha_style = att_module_style(query_style)
alpha_style = tf.cond(tf.equal(ctr_flag, 1), lambda: ctr_attention, lambda: alpha_style)
alpha_style = tf.Print(alpha_style, [alpha_style], message='alpha:', summarize=10)
context_style = tf.cond(tf.equal(ctr_flag, 1),
lambda: tf.reduce_sum(tf.expand_dims(alpha_style, axis=-1) * self.style_token, axis=1),
lambda: context_style)
context_style = tf.Print(context_style, [context_style], message='style:', summarize=10)
# alpha_style = ctr_attention
# alpha_style = tf.Print(alpha_style, [alpha_style], message='alpha', summarize=20)
# context_style = tf.reduce_sum(tf.expand_dims(alpha_style, axis=-1) * self.style_token, axis=1)
# context_style = tf.Print(context_style, [context_style], message='ctxt_style', summarize=20)
new_alpha_style_ta = old_alpha_style_ta.write(this_time, alpha_style)
with tf.variable_scope("weighting"):
weight_input = tf.concat([static_encoder_output, dec_pre_ed_inp], axis=-1)
weighting = tf.layers.dense(weight_input, 2, tf.nn.sigmoid)
# weighting = tf.Print(weighting, [weighting[1]], message='weighting')
weighting = tf.nn.softmax(weighting)
weight_text, weight_style = tf.split(weighting, [1, 1], -1)
# weight_text = tf.Print(weight_text, [weight_text], message='weight_text:', summarize=20)
weight_style = tf.Print(weight_style, [weight_style], message='weight_style:')
new_weight_ta = old_weight_ta.write(this_time, weight_text)
with tf.variable_scope('decoder_rnn'):
weighting_context = weight_text * context + weight_style * context_style
weight_per = tf.reduce_mean(tf.abs(weight_style * context_style) / (
tf.abs(weight_text * context) + tf.abs(weight_style * context_style)))
new_weight_per_ta = old_weight_per_ta.write(this_time, weight_per)
dec_input = tf.layers.dense(tf.concat([att_cell_out, weighting_context], axis=-1), 256)
# dec_input = tf.layers.dense(tf.concat([att_cell_out, context], axis=-1), 256)
dec_cell_out, dec_cell_state = dec_cell(dec_input, old_state_tup[1])
dense_out = tf.layers.dense(dec_cell_out, self.hyper_params.seq2seq_dim * reduc)
new_output_ta = old_output_ta.write(this_time, dense_out)
new_output = dense_out[:, -self.hyper_params.seq2seq_dim:]
new_state_tup = tuple([att_cell_state, dec_cell_state])
return tf.add(this_time, 1), new_output_ta, new_alpha_ta, new_alpha_style_ta, new_weight_ta,\
new_weight_per_ta, new_state_tup, context, context_style, new_output
# run loop
_, seq2seq_output_ta, alpha_ta, alpha_style_ta, weight_ta, weight_per_ta, *_ = tf.while_loop(cond, body, [init_time,
init_output_ta,
init_alpha_ta,
init_alpha_style_ta,
init_weight_ta,
init_weight_per_ta,
init_state_tup,
init_context,
init_context_style,
go_array
])
with tf.variable_scope('reshape_decode'):
seq2seq_output = tf.reshape(seq2seq_output_ta.stack(),
shape=(reduced_time_steps, batch_size, self.hyper_params.seq2seq_dim * reduc))
seq2seq_output = tf.reshape(tf.transpose(seq2seq_output, perm=(1, 0, 2)),
shape=(batch_size, output_time_steps, self.hyper_params.seq2seq_dim))
self.seq2seq_output = seq2seq_output
alpha_output = tf.reshape(alpha_ta.stack(),
shape=(reduced_time_steps, batch_size, input_time_steps))
alpha_output = tf.expand_dims(tf.transpose(alpha_output, perm=(1, 0, 2)), -1)
self.alpha_output = alpha_output
alpha_output_style = tf.reshape(alpha_style_ta.stack(),
shape=(reduced_time_steps, batch_size, self.hyper_params.styles_kind))
alpha_output_style = tf.expand_dims(tf.transpose(alpha_output_style, perm=(1, 0, 2)), -1) # batch major
self.alpha_output_style = alpha_output_style
weight_ta = tf.reshape(weight_ta.stack(), shape=(reduced_time_steps, batch_size, 1))
weight_ta = tf.transpose(weight_ta, perm=(1, 0, 2))
self.weight_ta = weight_ta
weight_per_ta = tf.reshape(weight_per_ta.stack(), shape=(reduced_time_steps, 1))
self.weight_per_ta = weight_per_ta
### Decoder [end]
### PostNet [begin]
post_output = modules.cbhg(seq2seq_output, training=False, k=8, bank_filters=128,
projection_filters=(256, self.hyper_params.seq2seq_dim),
highway_layers=4, highway_units=128,
bi_gru_units=128, sequence_length=None,
name='decoder_cbhg', reuse=False)
post_output = tf.layers.dense(post_output, self.hyper_params.post_dim, name='post_linear_transform')
self.post_output = post_output
def build_decoder_cell(self, encoder_outputs, encoder_state):
"""
构建解码器cell
:param encoder_outputs:
:param encoder_state:
:return:
"""
encoder_input_length = self.encoder_inputs_length
batch_size = self.batch_size
if self.bidirection:
encoder_state = encoder_state[-self.depth:]
if self.time_major:
encoder_outputs = tf.transpose(encoder_outputs, (1, 0, 2))
if self.use_beamsearch_decode:
# 复制多份
encoder_outputs = seq2seq.tile_batch(
encoder_outputs, multiplier=self.beam_width
)
encoder_state = seq2seq.tile_batch(
encoder_state, multiplier=self.beam_width
)
encoder_input_length = seq2seq.tile_batch(
self.encoder_inputs_length, multiplier=self.beam_width
)
batch_size *= self.beam_width
if self.attention_type.lower() == 'luong':
self.attention_mechanism = LuongAttention(
num_units=self.hidden_size,
memory=encoder_outputs,
memory_sequence_length=encoder_input_length
)
else:
self.attention_mechanism = BahdanauAttention(
num_units=self.hidden_size,
memory=encoder_outputs,
memory_sequence_length=encoder_input_length
)
cell = MultiRNNCell([
self.build_single_cell(
self.hidden_size,
use_residual=self.use_residual)
for _ in range(self.depth)
])
alignment_history = (
self.mode != 'train' and not self.use_beamsearch_decode
)
def cell_input_fn(inputs, attention):
if not self.use_residual:
return array_ops.concat([inputs, attention], -1)
attn_projection = layers.Dense(self.hidden_size,
dtype=tf.float32,
use_bias=False,
name='attention_cell_input_fn')
return attn_projection(array_ops.concat([inputs, attention], -1))
cell = AttentionWrapper(
cell=cell,
attention_mechanism=self.attention_mechanism,
attention_layer_size=self.hidden_size,
alignment_history=alignment_history,
cell_input_fn=cell_input_fn,
name='Attention_Wrapper'
)
decoder_initial_state = cell.zero_state(
batch_size, tf.float32)
# 传递encoder状态
decoder_initial_state = decoder_initial_state.clone(
cell_state=encoder_state
)
return cell, decoder_initial_state
def __init__(self, is_training, config, input_):
self._is_training = is_training
self._input = input_
self.batch_size = input_.batch_size
self.num_steps = input_.num_steps
self._input_data = input_.input_data
size = config.X_dim
hidden_size = config.hidden_size
vocab_size = config.vocab_size
self._targets = input_.targets
# Construct prior
prior = Prior(config.prior_pi, config.log_sigma1, config.log_sigma2)
# Fetch embeddings
inputs = input_.input_data
# Build the BBB LSTM cells
cells = []
for i in range(config.num_layers):
if (i == 0):
LSTM_input_size = config.X_dim
else:
LSTM_input_size = config.hidden_size
cells.append(
BayesianLSTMCell(LSTM_input_size,
config.hidden_size,
prior,
is_training,
forget_bias=0.0,
name="bbb_lstm_{}".format(i)))
cell = MultiRNNCell(cells, state_is_tuple=True)
self._initial_state = cell.zero_state(config.batch_size, data_type())
state = self._initial_state
# Forward pass for the truncated mini-batch
outputs = []
with tf.variable_scope("RNN"):
for time_step in range(self.num_steps):
if time_step > 0: tf.get_variable_scope().reuse_variables()
(cell_output, state) = cell(inputs[:, time_step, :], state)
outputs.append(cell_output)
output = tf.reshape(tf.concat(outputs, 1), [-1, hidden_size])
# Softmax weights
softmax_w = sample_posterior((hidden_size, vocab_size), "softmax_w",
prior, is_training)
softmax_b = sample_posterior((vocab_size, 1), "softmax_b", prior,
is_training)
logits = tf.nn.xw_plus_b(output, softmax_w, tf.squeeze(softmax_b))
logits = tf.reshape(logits,
[self.batch_size, self.num_steps, vocab_size])
self._output = tf.nn.softmax(logits)
loss = tf.contrib.seq2seq.sequence_loss(
logits,
input_.targets,
tf.ones([self.batch_size, self.num_steps], dtype=data_type()),
average_across_timesteps=False,
average_across_batch=False)
# Update the cost
# Remember to divide by batch size
self._cost = tf.reduce_sum(loss) / self.batch_size
self._kl_loss = 0.
self._final_state = state
if not is_training:
return
#Compute KL divergence
#B = number of batches aka the epoch size
#C = number of truncated sequences in a batch aka batch_size variable
B = self._input.epoch_size
C = self.batch_size
kl_loss = tf.add_n(tf.get_collection("KL_layers"), "kl_divergence")
kl_factor = 1.0 / (B * C)
self._kl_loss = kl_factor * kl_loss
self._total_loss = self._cost + self._kl_loss
self._lr = tf.Variable(0.0, trainable=False)
tvars = tf.trainable_variables()
grads, _ = tf.clip_by_global_norm(
tf.gradients(self._total_loss, tvars), config.max_grad_norm)
optimizer = tf.train.GradientDescentOptimizer(self._lr)
self._train_op = optimizer.apply_gradients(
zip(grads, tvars),
global_step=tf.contrib.framework.get_or_create_global_step())
self._new_lr = tf.placeholder(data_type(),
shape=[],
name="new_learning_rate")
self._lr_update = tf.assign(self._lr, self._new_lr)
class RNN(object):
def __init__(self, config, batch_size, seq_len, **kwargs):
assert config.name is not None
assert config.num_hidden > 0
assert config.num_layers > 0
assert config.num_classes > 0
self._config = config
self._batch_size = batch_size
self._seq_len = seq_len
self._input_ids = None
self._target_ids = None
self._stack = None
self._state = None
self._probs = None
self._loss = None
dropout = kwargs.get('dropout', 0)
assert 0 <= dropout < 1
self._build_network(dropout)
def _build_network(self, dropout):
# Legend for tensor shapes below:
# B := batch size
# C := number of classes
# H := number of hidden units (aka layer size)
# S := sequence length
# keep a reference to _config to make code below simpler
config = self._config
# Create size BxS input and target placeholder tensors
# These will be filled in with actual values at session runtime
data_dims = [self._batch_size, self._seq_len]
self._input_ids = tf.placeholder(tf.int32, data_dims)
self._target_ids = tf.placeholder(tf.int64, data_dims)
# Create an embedding tensor to represent integer inputs into H dimensions
# This must be done on the CPU, according to:
# https://github.com/tensorflow/tensorflow/blob/r0.7/tensorflow/examples/tutorials/word2vec/word2vec_basic.py#L143
# (Ops and variables pinned to the CPU because of missing GPU implementation)
with tf.device("/cpu:0"):
# embeddings is a CxH tensor
embeddings = tf.get_variable('embeddings', [config.num_classes, config.num_hidden])
# embedded is a BxSxH tensor
embedded = tf.nn.embedding_lookup(embeddings, self._input_ids)
# sequences is a list of length S containing Bx1xH tensors
sequences = tf.split(embedded, self._seq_len, 1)
# perform a "squeeze" on each item in the sequence list
# inputs is a list of length S containing BxH tensors
inputs = [tf.squeeze(seq, [1]) for seq in sequences]
# create LSTM cell and stack
cell = BasicLSTMCell(config.num_hidden)
if dropout > 0:
keep_prob = 1 - dropout
cell = DropoutWrapper(cell, output_keep_prob=keep_prob)
self._stack = MultiRNNCell([cell]*config.num_layers)
self._state = self._stack.zero_state(self._batch_size, tf.float32)
# Pump the inputs through the RNN layers
# outputs is a list of length S containing BxH tensors
outputs, self._state = static_rnn(self._stack, inputs, initial_state=self._state)
# assert len(outputs) == self._seq_len
#assert outputs[0].get_shape() == (self._batch_size, config.num_hidden), outputs[0].get_shape()
# Softmax weight tensor is HxC
W_soft = tf.get_variable('W_soft', [config.num_hidden, config.num_classes])
# Softmax bias tensor is Cx1
b_soft = tf.get_variable('b_soft', [config.num_classes])
# Reshape the output so that we can use it with the softmax weights and bias:
# - concat makes list into a BxSH tensor,
# - reshape converts the BxSH tensor into a BSxH tensor
output = tf.reshape(tf.concat(outputs, 1), [-1, config.num_hidden])
#assert output.get_shape() == (self._batch_size*self._seq_len, config.num_hidden), output.get_shape()
# logits is a (BSxH).(HxC) + 1xC = BSxC + 1xC = BSxC tensor
logits = tf.nn.xw_plus_b(output, W_soft, b_soft)
#assert logits.get_shape() == (self._batch_size*self._seq_len, config.num_classes), logits.get_shape()
# probs is a BSxC tensor, with entry (i,j) containing the probability that batch i is class j
self._probs = tf.nn.softmax(logits)
#assert self._probs.get_shape() == (self._batch_size*self._seq_len, config.num_classes), self._probs.get_shape()
# targets is a BSx1 tensor
targets = tf.reshape(self._target_ids, [self._batch_size*self._seq_len])
# cross_entropy is a BSx1 tensor
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=targets)
#assert cross_entropy.get_shape() == (self._batch_size*self._seq_len)
# loss is a scalar containing the mean of cross_entropy losses
self._loss = tf.reduce_mean(cross_entropy)
def reset_initial_state(self):
self._state = self._stack.zero_state(self._batch_size, tf.float32)
@property
def config(self):
return _config
@property
def inputs(self):
return self._input_ids
@inputs.setter
def inputs(self, input_ids):
self._input_ids = input_ids
@property
def targets(self):
return self._target_ids
@targets.setter
def targets(self, target_ids):
self._target_ids = target_ids
@property
def initial_state(self):
return self._state
@property
def probs(self):
return self._probs
@property
def loss(self):
return self._loss
