def __init__(self, rnn_size, rnn_layer, batch_size, input_embedding_size, dim_image, dim_hidden, max_words_q, vocabulary_size, drop_out_rate):
self.rnn_size = rnn_size
self.rnn_layer = rnn_layer
self.batch_size = batch_size
self.input_embedding_size = input_embedding_size
self.dim_image = dim_image
self.dim_hidden = dim_hidden
self.max_words_q = max_words_q
self.vocabulary_size = vocabulary_size
self.drop_out_rate = drop_out_rate
# question-embedding
self.embed_ques_W = tf.Variable(tf.random_uniform([self.vocabulary_size, self.input_embedding_size], -0.08, 0.08), name='embed_ques_W')
# encoder: RNN body
self.lstm_1 = core_rnn_cell.LSTMCell(rnn_size, input_embedding_size, use_peepholes=True, state_is_tuple = False)
self.lstm_dropout_1 = core_rnn_cell.DropoutWrapper(self.lstm_1, output_keep_prob = 1 - self.drop_out_rate)
self.lstm_2 = core_rnn_cell.LSTMCell(rnn_size, rnn_size, use_peepholes=True, state_is_tuple = False)
self.lstm_dropout_2 = core_rnn_cell.DropoutWrapper(self.lstm_2, output_keep_prob = 1 - self.drop_out_rate)
self.stacked_lstm = core_rnn_cell.MultiRNNCell([self.lstm_dropout_1, self.lstm_dropout_2], state_is_tuple = False)
# state-embedding
self.embed_state_W = tf.Variable(tf.random_uniform([2*rnn_size*rnn_layer, self.dim_hidden], -0.08,0.08),name='embed_state_W')
self.embed_state_b = tf.Variable(tf.random_uniform([self.dim_hidden], -0.08, 0.08), name='embed_state_b')
# image-embedding
self.embed_image_W = tf.Variable(tf.random_uniform([dim_image, self.dim_hidden], -0.08, 0.08), name='embed_image_W')
self.embed_image_b = tf.Variable(tf.random_uniform([dim_hidden], -0.08, 0.08), name='embed_image_b')
# score-embedding
self.embed_scor_W = tf.Variable(tf.random_uniform([dim_hidden, num_output], -0.08, 0.08), name='embed_scor_W')
self.embed_scor_b = tf.Variable(tf.random_uniform([num_output], -0.08, 0.08), name='embed_scor_b')
def _testDynamicDecodeRNNWithBasicTrainingSamplerMatchesDynamicRNN(
self, use_sequence_length):
sequence_length = [3, 4, 3, 1, 0]
batch_size = 5
max_time = 8
input_depth = 7
cell_depth = 10
max_out = max(sequence_length)
with self.test_session() as sess:
inputs = np.random.randn(batch_size, max_time,
input_depth).astype(np.float32)
cell = core_rnn_cell.LSTMCell(cell_depth)
zero_state = cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size)
sampler = sampling_decoder.BasicTrainingSampler(
inputs, sequence_length)
my_decoder = sampling_decoder.BasicSamplingDecoder(
cell=cell, sampler=sampler, initial_state=zero_state)
# Match the variable scope of dynamic_rnn below so we end up
# using the same variables
with vs.variable_scope("root") as scope:
final_decoder_outputs, final_decoder_state = decoder.dynamic_decode_rnn(
my_decoder,
# impute_finished=True ensures outputs and final state
# match those of dynamic_rnn called with sequence_length not None
impute_finished=use_sequence_length,
scope=scope)
with vs.variable_scope(scope, reuse=True) as scope:
final_rnn_outputs, final_rnn_state = rnn.dynamic_rnn(
cell,
inputs,
sequence_length=sequence_length
if use_sequence_length else None,
initial_state=zero_state,
scope=scope)
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"final_decoder_outputs": final_decoder_outputs,
"final_decoder_state": final_decoder_state,
"final_rnn_outputs": final_rnn_outputs,
"final_rnn_state": final_rnn_state
})
# Decoder only runs out to max_out; ensure values are identical
# to dynamic_rnn, which also zeros out outputs and passes along state.
self.assertAllClose(
sess_results["final_decoder_outputs"].rnn_output,
sess_results["final_rnn_outputs"][:, 0:max_out, :])
if use_sequence_length:
self.assertAllClose(sess_results["final_decoder_state"],
sess_results["final_rnn_state"])
def _testDynamicDecodeRNN(self, time_major):
sequence_length = [3, 4, 3, 1, 0]
batch_size = 5
max_time = 8
input_depth = 7
cell_depth = 10
max_out = max(sequence_length)
with self.test_session() as sess:
if time_major:
inputs = np.random.randn(max_time, batch_size,
input_depth).astype(np.float32)
else:
inputs = np.random.randn(batch_size, max_time,
input_depth).astype(np.float32)
cell = core_rnn_cell.LSTMCell(cell_depth)
sampler = sampling_decoder.BasicTrainingSampler(
inputs, sequence_length, time_major=time_major)
my_decoder = sampling_decoder.BasicSamplingDecoder(
cell=cell,
sampler=sampler,
initial_state=cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size))
final_outputs, final_state = decoder.dynamic_decode_rnn(
my_decoder, output_time_major=time_major)
def _t(shape):
if time_major:
return (shape[1], shape[0]) + shape[2:]
return shape
self.assertTrue(
isinstance(final_outputs,
sampling_decoder.SamplingDecoderOutput))
self.assertTrue(
isinstance(final_state, core_rnn_cell.LSTMStateTuple))
self.assertEqual(
_t((batch_size, None, cell_depth)),
tuple(final_outputs.rnn_output.get_shape().as_list()))
self.assertEqual(
_t((batch_size, None)),
tuple(final_outputs.sample_id.get_shape().as_list()))
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"final_outputs": final_outputs,
"final_state": final_state
})
self.assertEqual(_t((batch_size, max_out, cell_depth)),
sess_results["final_outputs"].rnn_output.shape)
self.assertEqual(_t((batch_size, max_out)),
sess_results["final_outputs"].sample_id.shape)
def _testDynamicDecodeRNN(self, time_major, has_attention):
encoder_sequence_length = [3, 2, 3, 1, 0]
decoder_sequence_length = [2, 0, 1, 2, 3]
batch_size = 5
decoder_max_time = 4
input_depth = 7
cell_depth = 9
attention_depth = 6
vocab_size = 20
end_token = vocab_size - 1
start_token = 0
embedding_dim = 50
max_out = max(decoder_sequence_length)
output_layer = layers_core.Dense(vocab_size,
use_bias=True,
activation=None)
beam_width = 3
with self.test_session() as sess:
embedding = np.random.randn(vocab_size,
embedding_dim).astype(np.float32)
cell = core_rnn_cell.LSTMCell(cell_depth)
if has_attention:
inputs = np.random.randn(batch_size, decoder_max_time,
input_depth).astype(np.float32)
attention_mechanism = attention_wrapper.BahdanauAttention(
num_units=attention_depth,
memory=inputs,
memory_sequence_length=encoder_sequence_length)
cell = attention_wrapper.AttentionWrapper(
cell=cell,
attention_mechanism=attention_mechanism,
attention_size=attention_depth,
alignment_history=False)
cell_state = cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size * beam_width)
bsd = beam_search_decoder.BeamSearchDecoder(
cell=cell,
embedding=embedding,
start_tokens=batch_size * [start_token],
end_token=end_token,
initial_state=cell_state,
beam_width=beam_width,
output_layer=output_layer,
length_penalty_weight=0.0)
final_outputs, final_state, final_sequence_lengths = (
decoder.dynamic_decode(bsd,
output_time_major=time_major,
maximum_iterations=max_out))
def _t(shape):
if time_major:
return (shape[1], shape[0]) + shape[2:]
return shape
self.assertTrue(
isinstance(final_outputs,
beam_search_decoder.FinalBeamSearchDecoderOutput))
self.assertTrue(
isinstance(final_state,
beam_search_decoder.BeamSearchDecoderState))
beam_search_decoder_output = final_outputs.beam_search_decoder_output
self.assertEqual(
_t((batch_size, None, beam_width)),
tuple(beam_search_decoder_output.scores.get_shape().as_list()))
self.assertEqual(
_t((batch_size, None, beam_width)),
tuple(final_outputs.predicted_ids.get_shape().as_list()))
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
'final_outputs':
final_outputs,
'final_state':
final_state,
'final_sequence_lengths':
final_sequence_lengths
})
max_sequence_length = np.max(
sess_results['final_sequence_lengths'])
# A smoke test
self.assertEqual(
_t((batch_size, max_sequence_length, beam_width)),
sess_results['final_outputs'].beam_search_decoder_output.
scores.shape)
self.assertEqual(
_t((batch_size, max_sequence_length, beam_width)),
sess_results['final_outputs'].beam_search_decoder_output.
predicted_ids.shape)
def _testStepWithTrainingHelper(self, use_output_layer):
sequence_length = [3, 4, 3, 1, 0]
batch_size = 5
max_time = 8
input_depth = 7
cell_depth = 10
output_layer_depth = 3
with self.test_session(use_gpu=True) as sess:
inputs = np.random.randn(batch_size, max_time,
input_depth).astype(np.float32)
cell = core_rnn_cell.LSTMCell(cell_depth)
helper = helper_py.TrainingHelper(inputs,
sequence_length,
time_major=False)
if use_output_layer:
output_layer = layers_core.Dense(output_layer_depth,
use_bias=False)
expected_output_depth = output_layer_depth
else:
output_layer = None
expected_output_depth = cell_depth
my_decoder = basic_decoder.BasicDecoder(
cell=cell,
helper=helper,
initial_state=cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size),
output_layer=output_layer)
output_size = my_decoder.output_size
output_dtype = my_decoder.output_dtype
self.assertEqual(
basic_decoder.BasicDecoderOutput(expected_output_depth,
tensor_shape.TensorShape([])),
output_size)
self.assertEqual(
basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
output_dtype)
(first_finished, first_inputs,
first_state) = my_decoder.initialize()
(step_outputs, step_state, step_next_inputs,
step_finished) = my_decoder.step(constant_op.constant(0),
first_inputs, first_state)
batch_size_t = my_decoder.batch_size
self.assertTrue(
isinstance(first_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
self.assertEqual((batch_size, expected_output_depth),
step_outputs[0].get_shape())
self.assertEqual((batch_size, ), step_outputs[1].get_shape())
self.assertEqual((batch_size, cell_depth),
first_state[0].get_shape())
self.assertEqual((batch_size, cell_depth),
first_state[1].get_shape())
self.assertEqual((batch_size, cell_depth),
step_state[0].get_shape())
self.assertEqual((batch_size, cell_depth),
step_state[1].get_shape())
if use_output_layer:
# The output layer was accessed
self.assertEqual(len(output_layer.variables), 1)
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"batch_size": batch_size_t,
"first_finished": first_finished,
"first_inputs": first_inputs,
"first_state": first_state,
"step_outputs": step_outputs,
"step_state": step_state,
"step_next_inputs": step_next_inputs,
"step_finished": step_finished
})
self.assertAllEqual([False, False, False, False, True],
sess_results["first_finished"])
self.assertAllEqual([False, False, False, True, True],
sess_results["step_finished"])
self.assertAllEqual(
np.argmax(sess_results["step_outputs"].rnn_output, -1),
sess_results["step_outputs"].sample_id)
def _testStepWithScheduledOutputTrainingHelper(self, sampling_probability,
use_next_input_layer,
use_auxiliary_inputs):
sequence_length = [3, 4, 3, 1, 0]
batch_size = 5
max_time = 8
input_depth = 7
cell_depth = input_depth
if use_next_input_layer:
cell_depth = 6
if use_auxiliary_inputs:
auxiliary_input_depth = 4
auxiliary_inputs = np.random.randn(
batch_size, max_time, auxiliary_input_depth).astype(np.float32)
else:
auxiliary_inputs = None
with self.test_session(use_gpu=True) as sess:
inputs = np.random.randn(batch_size, max_time,
input_depth).astype(np.float32)
cell = core_rnn_cell.LSTMCell(cell_depth)
sampling_probability = constant_op.constant(sampling_probability)
next_input_layer = None
if use_next_input_layer:
next_input_layer = layers_core.Dense(input_depth,
use_bias=False)
helper = helper_py.ScheduledOutputTrainingHelper(
inputs=inputs,
sequence_length=sequence_length,
sampling_probability=sampling_probability,
time_major=False,
next_input_layer=next_input_layer,
auxiliary_inputs=auxiliary_inputs)
my_decoder = basic_decoder.BasicDecoder(
cell=cell,
helper=helper,
initial_state=cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size))
output_size = my_decoder.output_size
output_dtype = my_decoder.output_dtype
self.assertEqual(
basic_decoder.BasicDecoderOutput(cell_depth,
tensor_shape.TensorShape([])),
output_size)
self.assertEqual(
basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
output_dtype)
(first_finished, first_inputs,
first_state) = my_decoder.initialize()
(step_outputs, step_state, step_next_inputs,
step_finished) = my_decoder.step(constant_op.constant(0),
first_inputs, first_state)
if use_next_input_layer:
output_after_next_input_layer = next_input_layer(
step_outputs.rnn_output)
batch_size_t = my_decoder.batch_size
self.assertTrue(
isinstance(first_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
self.assertEqual((batch_size, cell_depth),
step_outputs[0].get_shape())
self.assertEqual((batch_size, ), step_outputs[1].get_shape())
self.assertEqual((batch_size, cell_depth),
first_state[0].get_shape())
self.assertEqual((batch_size, cell_depth),
first_state[1].get_shape())
self.assertEqual((batch_size, cell_depth),
step_state[0].get_shape())
self.assertEqual((batch_size, cell_depth),
step_state[1].get_shape())
sess.run(variables.global_variables_initializer())
fetches = {
"batch_size": batch_size_t,
"first_finished": first_finished,
"first_inputs": first_inputs,
"first_state": first_state,
"step_outputs": step_outputs,
"step_state": step_state,
"step_next_inputs": step_next_inputs,
"step_finished": step_finished
}
if use_next_input_layer:
fetches[
"output_after_next_input_layer"] = output_after_next_input_layer
sess_results = sess.run(fetches)
self.assertAllEqual([False, False, False, False, True],
sess_results["first_finished"])
self.assertAllEqual([False, False, False, True, True],
sess_results["step_finished"])
sample_ids = sess_results["step_outputs"].sample_id
batch_where_not_sampling = np.where(np.logical_not(sample_ids))
batch_where_sampling = np.where(sample_ids)
auxiliary_inputs_to_concat = (
auxiliary_inputs[:, 1] if use_auxiliary_inputs else np.array(
[]).reshape(batch_size, 0).astype(np.float32))
expected_next_sampling_inputs = np.concatenate(
(sess_results["output_after_next_input_layer"]
[batch_where_sampling] if use_next_input_layer else
sess_results["step_outputs"].rnn_output[batch_where_sampling],
auxiliary_inputs_to_concat[batch_where_sampling]),
axis=-1)
self.assertAllClose(
sess_results["step_next_inputs"][batch_where_sampling],
expected_next_sampling_inputs)
self.assertAllClose(
sess_results["step_next_inputs"][batch_where_not_sampling],
np.concatenate(
(np.squeeze(inputs[batch_where_not_sampling, 1], axis=0),
auxiliary_inputs_to_concat[batch_where_not_sampling]),
axis=-1))
def testStepWithScheduledEmbeddingTrainingHelper(self):
sequence_length = [3, 4, 3, 1, 0]
batch_size = 5
max_time = 8
input_depth = 7
vocabulary_size = 10
with self.test_session(use_gpu=True) as sess:
inputs = np.random.randn(batch_size, max_time,
input_depth).astype(np.float32)
embeddings = np.random.randn(vocabulary_size,
input_depth).astype(np.float32)
half = constant_op.constant(0.5)
cell = core_rnn_cell.LSTMCell(vocabulary_size)
helper = helper_py.ScheduledEmbeddingTrainingHelper(
inputs=inputs,
sequence_length=sequence_length,
embedding=embeddings,
sampling_probability=half,
time_major=False)
my_decoder = basic_decoder.BasicDecoder(
cell=cell,
helper=helper,
initial_state=cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size))
output_size = my_decoder.output_size
output_dtype = my_decoder.output_dtype
self.assertEqual(
basic_decoder.BasicDecoderOutput(vocabulary_size,
tensor_shape.TensorShape([])),
output_size)
self.assertEqual(
basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
output_dtype)
(first_finished, first_inputs,
first_state) = my_decoder.initialize()
(step_outputs, step_state, step_next_inputs,
step_finished) = my_decoder.step(constant_op.constant(0),
first_inputs, first_state)
batch_size_t = my_decoder.batch_size
self.assertTrue(
isinstance(first_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
self.assertEqual((batch_size, vocabulary_size),
step_outputs[0].get_shape())
self.assertEqual((batch_size, ), step_outputs[1].get_shape())
self.assertEqual((batch_size, vocabulary_size),
first_state[0].get_shape())
self.assertEqual((batch_size, vocabulary_size),
first_state[1].get_shape())
self.assertEqual((batch_size, vocabulary_size),
step_state[0].get_shape())
self.assertEqual((batch_size, vocabulary_size),
step_state[1].get_shape())
self.assertEqual((batch_size, input_depth),
step_next_inputs.get_shape())
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"batch_size": batch_size_t,
"first_finished": first_finished,
"first_inputs": first_inputs,
"first_state": first_state,
"step_outputs": step_outputs,
"step_state": step_state,
"step_next_inputs": step_next_inputs,
"step_finished": step_finished
})
self.assertAllEqual([False, False, False, False, True],
sess_results["first_finished"])
self.assertAllEqual([False, False, False, True, True],
sess_results["step_finished"])
sample_ids = sess_results["step_outputs"].sample_id
batch_where_not_sampling = np.where(sample_ids == -1)
batch_where_sampling = np.where(sample_ids > -1)
self.assertAllClose(
sess_results["step_next_inputs"][batch_where_sampling],
embeddings[sample_ids[batch_where_sampling]])
self.assertAllClose(
sess_results["step_next_inputs"][batch_where_not_sampling],
np.squeeze(inputs[batch_where_not_sampling, 1]))
def testStepWithGreedyEmbeddingHelper(self):
batch_size = 5
vocabulary_size = 7
cell_depth = vocabulary_size # cell's logits must match vocabulary size
input_depth = 10
start_tokens = [0] * batch_size
end_token = 1
with self.test_session(use_gpu=True) as sess:
embeddings = np.random.randn(vocabulary_size,
input_depth).astype(np.float32)
cell = core_rnn_cell.LSTMCell(vocabulary_size)
helper = helper_py.GreedyEmbeddingHelper(embeddings, start_tokens,
end_token)
my_decoder = basic_decoder.BasicDecoder(
cell=cell,
helper=helper,
initial_state=cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size))
output_size = my_decoder.output_size
output_dtype = my_decoder.output_dtype
self.assertEqual(
basic_decoder.BasicDecoderOutput(cell_depth,
tensor_shape.TensorShape([])),
output_size)
self.assertEqual(
basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32),
output_dtype)
(first_finished, first_inputs,
first_state) = my_decoder.initialize()
(step_outputs, step_state, step_next_inputs,
step_finished) = my_decoder.step(constant_op.constant(0),
first_inputs, first_state)
batch_size_t = my_decoder.batch_size
self.assertTrue(
isinstance(first_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_outputs, basic_decoder.BasicDecoderOutput))
self.assertEqual((batch_size, cell_depth),
step_outputs[0].get_shape())
self.assertEqual((batch_size, ), step_outputs[1].get_shape())
self.assertEqual((batch_size, cell_depth),
first_state[0].get_shape())
self.assertEqual((batch_size, cell_depth),
first_state[1].get_shape())
self.assertEqual((batch_size, cell_depth),
step_state[0].get_shape())
self.assertEqual((batch_size, cell_depth),
step_state[1].get_shape())
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"batch_size": batch_size_t,
"first_finished": first_finished,
"first_inputs": first_inputs,
"first_state": first_state,
"step_outputs": step_outputs,
"step_state": step_state,
"step_next_inputs": step_next_inputs,
"step_finished": step_finished
})
expected_sample_ids = np.argmax(
sess_results["step_outputs"].rnn_output, -1)
expected_step_finished = (expected_sample_ids == end_token)
expected_step_next_inputs = embeddings[expected_sample_ids]
self.assertAllEqual([False, False, False, False, False],
sess_results["first_finished"])
self.assertAllEqual(expected_step_finished,
sess_results["step_finished"])
self.assertAllEqual(expected_sample_ids,
sess_results["step_outputs"].sample_id)
self.assertAllEqual(expected_step_next_inputs,
sess_results["step_next_inputs"])
def _testDynamicDecodeRNN(self, time_major, maximum_iterations=None):
sequence_length = [3, 4, 3, 1, 0]
batch_size = 5
max_time = 8
input_depth = 7
cell_depth = 10
max_out = max(sequence_length)
with self.test_session(use_gpu=True) as sess:
if time_major:
inputs = np.random.randn(max_time, batch_size,
input_depth).astype(np.float32)
else:
inputs = np.random.randn(batch_size, max_time,
input_depth).astype(np.float32)
cell = core_rnn_cell.LSTMCell(cell_depth)
helper = helper_py.TrainingHelper(inputs,
sequence_length,
time_major=time_major)
my_decoder = basic_decoder.BasicDecoder(
cell=cell,
helper=helper,
initial_state=cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size))
final_outputs, final_state, final_sequence_length = (
decoder.dynamic_decode(my_decoder,
output_time_major=time_major,
maximum_iterations=maximum_iterations))
def _t(shape):
if time_major:
return (shape[1], shape[0]) + shape[2:]
return shape
self.assertTrue(
isinstance(final_outputs, basic_decoder.BasicDecoderOutput))
self.assertTrue(
isinstance(final_state, core_rnn_cell.LSTMStateTuple))
self.assertEqual(
(batch_size, ),
tuple(final_sequence_length.get_shape().as_list()))
self.assertEqual(
_t((batch_size, None, cell_depth)),
tuple(final_outputs.rnn_output.get_shape().as_list()))
self.assertEqual(
_t((batch_size, None)),
tuple(final_outputs.sample_id.get_shape().as_list()))
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"final_outputs":
final_outputs,
"final_state":
final_state,
"final_sequence_length":
final_sequence_length,
})
# Mostly a smoke test
time_steps = max_out
if maximum_iterations is not None:
time_steps = min(max_out, maximum_iterations)
self.assertEqual(_t((batch_size, time_steps, cell_depth)),
sess_results["final_outputs"].rnn_output.shape)
self.assertEqual(_t((batch_size, time_steps)),
sess_results["final_outputs"].sample_id.shape)
def _testWithAttention(self,
create_attention_mechanism,
expected_final_output,
expected_final_state,
attention_mechanism_depth=3,
alignment_history=False,
expected_final_alignment_history=None,
name=""):
encoder_sequence_length = [3, 2, 3, 1, 0]
decoder_sequence_length = [2, 0, 1, 2, 3]
batch_size = 5
encoder_max_time = 8
decoder_max_time = 4
input_depth = 7
encoder_output_depth = 10
cell_depth = 9
attention_depth = 6
decoder_inputs = np.random.randn(batch_size, decoder_max_time,
input_depth).astype(np.float32)
encoder_outputs = np.random.randn(batch_size, encoder_max_time,
encoder_output_depth).astype(
np.float32)
attention_mechanism = create_attention_mechanism(
num_units=attention_mechanism_depth,
memory=encoder_outputs,
memory_sequence_length=encoder_sequence_length)
with self.test_session(use_gpu=True) as sess:
with vs.variable_scope(
"root",
initializer=init_ops.random_normal_initializer(stddev=0.01,
seed=3)):
cell = core_rnn_cell.LSTMCell(cell_depth)
cell = wrapper.AttentionWrapper(
cell,
attention_mechanism,
attention_size=attention_depth,
alignment_history=alignment_history)
helper = helper_py.TrainingHelper(decoder_inputs,
decoder_sequence_length)
my_decoder = basic_decoder.BasicDecoder(
cell=cell,
helper=helper,
initial_state=cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size))
final_outputs, final_state = decoder.dynamic_decode(my_decoder)
self.assertTrue(
isinstance(final_outputs, basic_decoder.BasicDecoderOutput))
self.assertTrue(
isinstance(final_state, wrapper.AttentionWrapperState))
self.assertTrue(
isinstance(final_state.cell_state,
core_rnn_cell.LSTMStateTuple))
self.assertEqual(
(batch_size, None, attention_depth),
tuple(final_outputs.rnn_output.get_shape().as_list()))
self.assertEqual(
(batch_size, None),
tuple(final_outputs.sample_id.get_shape().as_list()))
self.assertEqual(
(batch_size, attention_depth),
tuple(final_state.attention.get_shape().as_list()))
self.assertEqual(
(batch_size, cell_depth),
tuple(final_state.cell_state.c.get_shape().as_list()))
self.assertEqual(
(batch_size, cell_depth),
tuple(final_state.cell_state.h.get_shape().as_list()))
if alignment_history:
state_alignment_history = final_state.alignment_history.stack()
# Remove the history from final_state for purposes of the
# remainder of the tests.
final_state = final_state._replace(alignment_history=()) # pylint: disable=protected-access
self.assertEqual(
(None, batch_size, encoder_max_time),
tuple(state_alignment_history.get_shape().as_list()))
else:
state_alignment_history = ()
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"final_outputs":
final_outputs,
"final_state":
final_state,
"state_alignment_history":
state_alignment_history,
})
print("Copy/paste (%s)\nexpected_final_output = " % name,
sess_results["final_outputs"])
sys.stdout.flush()
print("Copy/paste (%s)\nexpected_final_state = " % name,
sess_results["final_state"])
sys.stdout.flush()
print(
"Copy/paste (%s)\nexpected_final_alignment_history = " % name,
sess_results["state_alignment_history"])
sys.stdout.flush()
nest.map_structure(self.assertAllClose, expected_final_output,
sess_results["final_outputs"])
nest.map_structure(self.assertAllClose, expected_final_state,
sess_results["final_state"])
if alignment_history: # by default, the wrapper emits attention as output
self.assertAllClose(
# outputs are batch major but the stacked TensorArray is time major
sess_results["state_alignment_history"],
expected_final_alignment_history)
def _testWithAttention(self,
create_attention_mechanism,
expected_final_outputs,
expected_final_state,
attention_mechanism_depth=3):
encoder_sequence_length = [3, 2, 3, 1, 0]
decoder_sequence_length = [2, 0, 1, 2, 3]
batch_size = 5
encoder_max_time = 8
decoder_max_time = 4
input_depth = 7
encoder_output_depth = 10
cell_depth = 9
attention_depth = 6
decoder_inputs = np.random.randn(batch_size, decoder_max_time,
input_depth).astype(np.float32)
encoder_outputs = np.random.randn(batch_size, encoder_max_time,
encoder_output_depth).astype(
np.float32)
attention_mechanism = create_attention_mechanism(
num_units=attention_mechanism_depth,
memory=encoder_outputs,
memory_sequence_length=encoder_sequence_length)
with self.test_session() as sess:
with vs.variable_scope(
"root",
initializer=init_ops.random_normal_initializer(stddev=0.01,
seed=3)):
cell = core_rnn_cell.LSTMCell(cell_depth)
cell = wrapper.DynamicAttentionWrapper(
cell, attention_mechanism, attention_size=attention_depth)
helper = helper_py.TrainingHelper(decoder_inputs,
decoder_sequence_length)
my_decoder = basic_decoder.BasicDecoder(
cell=cell,
helper=helper,
initial_state=cell.zero_state(dtype=dtypes.float32,
batch_size=batch_size))
final_outputs, final_state = decoder.dynamic_decode(my_decoder)
self.assertTrue(
isinstance(final_outputs, basic_decoder.BasicDecoderOutput))
self.assertTrue(
isinstance(final_state, wrapper.DynamicAttentionWrapperState))
self.assertTrue(
isinstance(final_state.cell_state,
core_rnn_cell.LSTMStateTuple))
self.assertEqual(
(batch_size, None, attention_depth),
tuple(final_outputs.rnn_output.get_shape().as_list()))
self.assertEqual(
(batch_size, None),
tuple(final_outputs.sample_id.get_shape().as_list()))
self.assertEqual(
(batch_size, attention_depth),
tuple(final_state.attention.get_shape().as_list()))
self.assertEqual(
(batch_size, cell_depth),
tuple(final_state.cell_state.c.get_shape().as_list()))
self.assertEqual(
(batch_size, cell_depth),
tuple(final_state.cell_state.h.get_shape().as_list()))
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"final_outputs": final_outputs,
"final_state": final_state
})
nest.map_structure(self.assertAllClose, expected_final_outputs,
sess_results["final_outputs"])
nest.map_structure(self.assertAllClose, expected_final_state,
sess_results["final_state"])
def rnn(hnum1, hnum2, hnum3):
file = open(r"wind.csv")
file.readline() # 读掉第一行,下次再引用file的时候,将file的文件指针指向第二行开始的文件.
reader = csv.reader(file)
raw_data = []
for date, hors, u, v, ws, wd in reader:
if ws != 'NA':
raw_data.append(float(ws))
# difference = max(raw_data)-min(raw_data)
# raw_data = [i/difference for i in raw_data]
# raw_data=[10*math.sin(0.1*i) for i in range(20000)]
sequence_length = 100 # 代表以往数据的移动窗口宽度 注意取值范围 (可调参数)
predict_length = 16 # 代表预测数据的移动窗口宽度 即一次预测出的结果数 注意取值范围 (可调参数)
train_input_all = []
for i in range(0, len(raw_data[0:-sequence_length - predict_length + 1])):
temp_list = []
for j in range(sequence_length):
temp_list.append([raw_data[i + j]])
train_input_all.append(temp_list)
train_label_all = []
train_label_all1 = []
for i in range(sequence_length, len(raw_data) - predict_length + 1):
temp_list = []
for j in range(predict_length):
temp_list.append(raw_data[i + j])
train_label_all.append(temp_list)
train_label_all1.append(raw_data[i + j])
seperate_point = 5000 # 测试集与训练集分割点 （可调数）
test_point = 90000 # 使用的数据量大小（可调数 且必须大于seperate_point）
test_point_start = 80000
train_input = train_input_all[0:seperate_point]
test_input = train_input_all[test_point_start + 1:test_point]
train_output = train_label_all[0:seperate_point] # 训练数据标签格式1
train_output1 = train_label_all1[0:seperate_point] # 训练数据标签格式2
test_output = train_label_all[test_point_start + 1:test_point] # 测试数据标签格式1
test_output1 = train_label_all1[test_point_start +
1:test_point] # 测试数据标签格式2
# 打乱训练集
index = [i for i in range(len(train_input))]
shuffle(index)
train_input = [train_input[index[i]] for i in range(len(index))]
train_output = [train_output[index[i]] for i in range(len(index))]
data = tf.placeholder(
tf.float32, [None, sequence_length, 1]) # batch_size maxtime deepth
target = tf.placeholder(tf.float32, [None, predict_length], name='target')
num_hidden = [hnum1, hnum2, hnum3] # 隐含层数量(可调参数)
# cell = rnn_cell.BasicRNNCell(num_hidden)
# cells = rnn_cell.LSTMCell(num_hidden[0], state_is_tuple=True)
cell_layer1 = rnn_cell.LSTMCell(num_hidden[0], state_is_tuple=True)
# cell_layer1 = rnn_cell.DropoutWrapper(cell_layer1, input_keep_prob=0.5, output_keep_prob=0.5)
cell_layer2 = rnn_cell.LSTMCell(num_hidden[1], state_is_tuple=True)
# cell_layer2 = rnn_cell.DropoutWrapper(cell_layer2, input_keep_prob=0.5, output_keep_prob=0.5)
cell_layer3 = rnn_cell.LSTMCell(num_hidden[2], state_is_tuple=True)
# cell_layer4 = rnn_cell.LSTMCell(num_hidden[3], state_is_tuple=True)
# cell_layer5 = rnn_cell.LSTMCell(num_hidden[4], state_is_tuple=True)
cells = rnn_cell.MultiRNNCell([cell_layer1, cell_layer2,
cell_layer3]) # 建立多层rnn
val, state = tf.nn.dynamic_rnn(cells, data, dtype=tf.float32)
val = tf.transpose(val, [1, 0, 2])
val_shape = val.get_shape()
last = tf.gather(val, int(val.get_shape()[0]) - 1)
last_shape = last.get_shape()
weight = tf.Variable(
tf.truncated_normal([num_hidden[-1],
int(target.get_shape()[1])]))
bias = tf.Variable(tf.constant(0.1, shape=[target.get_shape()[1]]))
prediction = tf.matmul(last, weight) + bias
prediction_shape = prediction.get_shape()
loss = tf.reduce_mean(tf.square(prediction - target))
loss_shape = loss.get_shape()
optimizer = tf.train.AdamOptimizer()
minimize = optimizer.minimize(loss)
# mistakes = tf.not_equal(tf.argmax(target, 1), tf.argmax(prediction, 1))
error = tf.reduce_mean(tf.square(prediction - target))
error_sep = tf.square(prediction - target) # 计算每一个预测分量的误差
init_op = tf.global_variables_initializer()
sess = tf.Session()
saver = tf.train.Saver()
sess.run(init_op) # 在这里，可以执行这个语句，也可以不执行，即使执行了，初始化的值也会被restore的值给override
# saver.restore(sess, r"parameter_5.ckpt")
batch_size = 10 # （可调参数）
no_of_batches = int(len(train_input) / batch_size)
epoch = 25 # （可调参数）
total_error1 = 0
predict_result1 = []
total_error = 0
predict_error = []
predict_result = []
temp = 0 # 测试变量
for i in range(epoch):
ptr = 0
for j in range(no_of_batches):
inp, out = train_input[ptr:ptr +
batch_size], train_output[ptr:ptr +
batch_size]
ptr += batch_size
sess.run(minimize, {data: inp, target: out})
print("Epoch - ", str(i))
# sess.run(error, {data: train_input, target: train_output})
# 观察测试样本的估计情况
total_error = 0
predict_error = []
predict_result = []
temp = 0
temp1 = 0
temp_sep = []
total_error_sep = []
for i in range(len(test_input)):
inp, out = test_input[i:i + 1], test_output[i:i + 1]
temp1 = sess.run(prediction, {data: inp, target: out})
temp = sess.run(error, {data: inp, target: out})
temp_sep = sess.run(error_sep, {data: inp, target: out})
# print(temp1)
total_error += temp
# predict_error.append(temp)
predict_result.append(temp1[0])
total_error_sep.append(temp_sep)
total_error /= len(test_input)
total_error = math.sqrt(total_error)
total_error_sep = (np.array(total_error_sep)).mean(axis=0)
# 观察训练样本的训练情况
total_error1 = 0
predict_result1 = []
temp2 = 0
temp3 = 0
temp_sep1 = []
total_error_sep1 = []
for i in range(len(train_input)):
inp, out = train_input[i:i + 1], train_output[i:i + 1]
temp2 = sess.run(error, {data: inp, target: out})
temp3 = sess.run(prediction, {data: inp, target: out})
temp_sep1 = sess.run(error_sep, {data: inp, target: out})
total_error1 += temp2
predict_result1.append((temp3[0]))
total_error_sep1.append(temp_sep1)
total_error1 /= len(train_input)
total_error1 = math.sqrt(total_error1)
total_error_sep1 = (np.array(total_error_sep1)).mean(axis=0)
# incorrect = sess.run(error, {data: test_input, target: test_output})
print('Epoch {:2d} error {:3.5f}'.format(i + 1, total_error))
# print('predict_error')
# print(predict_error)
# print('predict_result')
# print(predict_result)
saver.save(sess, r"parameter_5.ckpt")
sess.close()
# saver = tf.train.Saver()
# pylab.plot(predict_result)# predict_result1是测试样本的检擦结果predict_result
# pylab.plot(test_output)
# pylab.plot(predict_result1)# predict_result1是训练样本的检擦结果predict_result
# pylab.plot(train_output)#train_output1是训练样本的检查结果test_output1
corrcoef_result_test = []
corrcoef_result_train = []
for cursor in range(16):
test_output_single = [
test_output[i][int(cursor)] for i in range(len(test_output))
]
predict_result_single = [
predict_result[i][int(cursor)] for i in range(len(predict_result))
]
corrcoef_result_test.append(
corrcoef(test_output_single, predict_result_single))
#print(corrcoef(test_output_single, predict_result_single))
for cursor in range(16):
train_output_single = [
train_output[i][int(cursor)] for i in range(len(train_output))
]
predict_result1_single = [
predict_result1[i][int(cursor)]
for i in range(len(predict_result1))
]
corrcoef_result_train.append(
corrcoef(train_output_single, predict_result1_single))
#print(corrcoef(train_output_single, predict_result1_single))
'''
需要记录的数据 1输入训练样本编号 2测试样本编号 3使用的模型类型 4使用的模型参数（隐含层数量 隐含层每层的单元数量）5训练batch大小
6训练的周期数 7预测结果 8输出误差大小 9输出的相关系数 10训练时间
'''
csvfile = open(r'short_result5.csv', 'a')
writer = csv.writer(csvfile)
writer.writerow([
'epoch', 'seperate_point', 'test_point_start', 'test_point',
'num_hidden', 'sequence_length', 'predict_length', 'batch_size'
])
data = [(epoch, seperate_point, test_point_start, test_point, num_hidden,
sequence_length, predict_length, batch_size)]
writer.writerows(data)
writer.writerow(['corrcoef_result_test'])
writer.writerow(corrcoef_result_test)
writer.writerow(['corrcoef_result_train'])
writer.writerow(corrcoef_result_train)
writer.writerow(['prediction_result'])
writer.writerow(['total_error_sep'])
for i in range(len(total_error_sep)):
writer.writerow(total_error_sep[i])
writer.writerow(['total_error'])
writer.writerow([total_error])
csvfile.close()
def testStepWithBasicTrainingSampler(self):
sequence_length = [3, 4, 3, 1, 0]
batch_size = 5
max_time = 8
input_depth = 7
cell_depth = 10
with self.test_session() as sess:
inputs = np.random.randn(batch_size, max_time,
input_depth).astype(np.float32)
cell = core_rnn_cell.LSTMCell(cell_depth)
sampler = sampling_decoder.BasicTrainingSampler(
inputs, sequence_length, time_major=False)
my_decoder = sampling_decoder.BasicSamplingDecoder(
cell=cell,
sampler=sampler,
initial_state=cell.zero_state(
dtype=dtypes.float32, batch_size=batch_size))
output_size = my_decoder.output_size
output_dtype = my_decoder.output_dtype
self.assertEqual(
sampling_decoder.SamplingDecoderOutput(cell_depth,
tensor_shape.TensorShape([])),
output_size)
self.assertEqual(
sampling_decoder.SamplingDecoderOutput(dtypes.float32, dtypes.int32),
output_dtype)
(first_finished, first_inputs, first_state) = my_decoder.initialize()
(step_outputs, step_state, step_next_inputs,
step_finished) = my_decoder.step(
constant_op.constant(0), first_inputs, first_state)
batch_size_t = my_decoder.batch_size
self.assertTrue(isinstance(first_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(isinstance(step_state, core_rnn_cell.LSTMStateTuple))
self.assertTrue(
isinstance(step_outputs, sampling_decoder.SamplingDecoderOutput))
self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape())
self.assertEqual((batch_size,), step_outputs[1].get_shape())
self.assertEqual((batch_size, cell_depth), first_state[0].get_shape())
self.assertEqual((batch_size, cell_depth), first_state[1].get_shape())
self.assertEqual((batch_size, cell_depth), step_state[0].get_shape())
self.assertEqual((batch_size, cell_depth), step_state[1].get_shape())
sess.run(variables.global_variables_initializer())
sess_results = sess.run({
"batch_size": batch_size_t,
"first_finished": first_finished,
"first_inputs": first_inputs,
"first_state": first_state,
"step_outputs": step_outputs,
"step_state": step_state,
"step_next_inputs": step_next_inputs,
"step_finished": step_finished
})
self.assertAllEqual([False, False, False, False, True],
sess_results["first_finished"])
self.assertAllEqual([False, False, False, True, True],
sess_results["step_finished"])
self.assertAllEqual(
np.argmax(sess_results["step_outputs"].rnn_output, -1),
sess_results["step_outputs"].sample_id)
'''
index = [i for i in range(len(test_input))]
shuffle(index)
test_input = [test_input[index[i]] for i in range(len(index))]
test_output = [test_output[index[i]] for i in range(len(index))]
'''
'''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''
''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''
data = tf.placeholder(tf.float32,
[None, sequence_length, 1]) #batch_size maxtime deepth
target = tf.placeholder(tf.float32, [None, predict_length], name='target')
num_hidden = [30, 30] #隐含层数量(可调参数)
#cell = rnn_cell.BasicRNNCell(num_hidden)
#cells = rnn_cell.LSTMCell(num_hidden[0], state_is_tuple=True)
cell_layer1 = rnn_cell.LSTMCell(num_hidden[0], state_is_tuple=True)
#cell_layer1 = rnn_cell.DropoutWrapper(cell_layer1, input_keep_prob=0.5, output_keep_prob=0.5)
cell_layer2 = rnn_cell.LSTMCell(num_hidden[1], state_is_tuple=True)
#cell_layer2 = rnn_cell.DropoutWrapper(cell_layer2, input_keep_prob=0.5, output_keep_prob=0.5)
#cell_layer3 = rnn_cell.LSTMCell(num_hidden[2], state_is_tuple=True)
#cell_layer4 = rnn_cell.LSTMCell(num_hidden[3], state_is_tuple=True)
#cell_layer5 = rnn_cell.LSTMCell(num_hidden[4], state_is_tuple=True)
cells = rnn_cell.MultiRNNCell([cell_layer1, cell_layer2]) #建立多层rnn
val, state = tf.nn.dynamic_rnn(cells, data, dtype=tf.float32)
val = tf.transpose(val, [1, 0, 2])
val_shape = val.get_shape()
last = tf.gather(val, int(val.get_shape()[0]) - 1)
def __init__(self, name, env_state_size, env_stack_size, num_actions,
hidden_size, num_stacks, batch_size, scope):
self.name = name
self.batch_size = batch_size
self.hidden_size = hidden_size
self.lstm = core_rnn_cell.LSTMCell(num_units=hidden_size)
self.zero_state = self.lstm.zero_state(batch_size=self.batch_size,
dtype=tf.float32)
self.rnn_state_ph = tf.placeholder(shape=[2, batch_size, hidden_size],
dtype=tf.float32)
self.rnn_state = self.to_lstm_state(self.rnn_state_ph)
self.env_state_input = tf.placeholder(
shape=[batch_size, None, env_state_size], dtype=tf.float32)
self.env_stack_input = tf.placeholder(shape=[
batch_size, None, num_stacks * env_stack_size * env_state_size
],
dtype=tf.float32)
self.action_softmax_div = tf.placeholder(shape=[None, 1],
dtype=tf.float32)
self.stack_softmax_div = tf.placeholder(shape=[None, 1],
dtype=tf.float32)
self.env_input = tf.concat(
(self.env_state_input, self.env_stack_input), axis=2)
hidden, self.hidden_state = rnn.dynamic_rnn(
self.lstm, self.env_input, initial_state=self.rnn_state)
out_act = slim.fully_connected(hidden,
num_actions,
activation_fn=None,
biases_initializer=None)
self.output = tf.nn.softmax(out_act / self.action_softmax_div)
self.chosen_action = tf.argmax(self.output, axis=2)
out_sta = slim.fully_connected(hidden,
num_stacks,
activation_fn=None,
biases_initializer=None)
self.out_stack = tf.nn.softmax(out_sta / self.stack_softmax_div)
self.chosen_stack = tf.argmax(self.out_stack, axis=2)
# The next six lines establish the training procedure. We feed the reward and chosen action into the network
# to compute the loss, and use it to update the network.
self.action_reward_holder = tf.placeholder(shape=[batch_size, None],
dtype=tf.float32)
self.stack_reward_holder = tf.placeholder(shape=[batch_size, None],
dtype=tf.float32)
self.action_holder = tf.placeholder(shape=[batch_size, None],
dtype=tf.int32)
self.stack_holder = tf.placeholder(shape=[batch_size, None],
dtype=tf.int32)
loss_actions, loss_stacks = tf.scan(
self.determine_input_seq,
elems=[
self.output, self.action_holder, self.out_stack,
self.stack_holder, self.action_reward_holder,
self.stack_reward_holder
],
initializer=(tf.constant(0.0), tf.constant(0.0)))
self.loss = loss_actions + loss_stacks
self.trainable_variables = tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope.name)
self.gradients = tf.gradients(self.loss, self.trainable_variables)
optimizer = tf.train.AdamOptimizer()
self.update = optimizer.apply_gradients(
zip(self.gradients, self.trainable_variables))
self.trained_variables = []
self._saver = tf.train.Saver(self.trainable_variables)
