def __init__(self, rnn_size, rnn_layer, batch_size, input_embedding_size, dim_image, dim_hidden, max_words_q, vocabulary_size, drop_out_rate): self.rnn_size = rnn_size self.rnn_layer = rnn_layer self.batch_size = batch_size self.input_embedding_size = input_embedding_size self.dim_image = dim_image self.dim_hidden = dim_hidden self.max_words_q = max_words_q self.vocabulary_size = vocabulary_size self.drop_out_rate = drop_out_rate # question-embedding self.embed_ques_W = tf.Variable(tf.random_uniform([self.vocabulary_size, self.input_embedding_size], -0.08, 0.08), name='embed_ques_W') # encoder: RNN body self.lstm_1 = core_rnn_cell.LSTMCell(rnn_size, input_embedding_size, use_peepholes=True, state_is_tuple = False) self.lstm_dropout_1 = core_rnn_cell.DropoutWrapper(self.lstm_1, output_keep_prob = 1 - self.drop_out_rate) self.lstm_2 = core_rnn_cell.LSTMCell(rnn_size, rnn_size, use_peepholes=True, state_is_tuple = False) self.lstm_dropout_2 = core_rnn_cell.DropoutWrapper(self.lstm_2, output_keep_prob = 1 - self.drop_out_rate) self.stacked_lstm = core_rnn_cell.MultiRNNCell([self.lstm_dropout_1, self.lstm_dropout_2], state_is_tuple = False) # state-embedding self.embed_state_W = tf.Variable(tf.random_uniform([2*rnn_size*rnn_layer, self.dim_hidden], -0.08,0.08),name='embed_state_W') self.embed_state_b = tf.Variable(tf.random_uniform([self.dim_hidden], -0.08, 0.08), name='embed_state_b') # image-embedding self.embed_image_W = tf.Variable(tf.random_uniform([dim_image, self.dim_hidden], -0.08, 0.08), name='embed_image_W') self.embed_image_b = tf.Variable(tf.random_uniform([dim_hidden], -0.08, 0.08), name='embed_image_b') # score-embedding self.embed_scor_W = tf.Variable(tf.random_uniform([dim_hidden, num_output], -0.08, 0.08), name='embed_scor_W') self.embed_scor_b = tf.Variable(tf.random_uniform([num_output], -0.08, 0.08), name='embed_scor_b')
def _testDynamicDecodeRNNWithBasicTrainingSamplerMatchesDynamicRNN( self, use_sequence_length): sequence_length = [3, 4, 3, 1, 0] batch_size = 5 max_time = 8 input_depth = 7 cell_depth = 10 max_out = max(sequence_length) with self.test_session() as sess: inputs = np.random.randn(batch_size, max_time, input_depth).astype(np.float32) cell = core_rnn_cell.LSTMCell(cell_depth) zero_state = cell.zero_state(dtype=dtypes.float32, batch_size=batch_size) sampler = sampling_decoder.BasicTrainingSampler( inputs, sequence_length) my_decoder = sampling_decoder.BasicSamplingDecoder( cell=cell, sampler=sampler, initial_state=zero_state) # Match the variable scope of dynamic_rnn below so we end up # using the same variables with vs.variable_scope("root") as scope: final_decoder_outputs, final_decoder_state = decoder.dynamic_decode_rnn( my_decoder, # impute_finished=True ensures outputs and final state # match those of dynamic_rnn called with sequence_length not None impute_finished=use_sequence_length, scope=scope) with vs.variable_scope(scope, reuse=True) as scope: final_rnn_outputs, final_rnn_state = rnn.dynamic_rnn( cell, inputs, sequence_length=sequence_length if use_sequence_length else None, initial_state=zero_state, scope=scope) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "final_decoder_outputs": final_decoder_outputs, "final_decoder_state": final_decoder_state, "final_rnn_outputs": final_rnn_outputs, "final_rnn_state": final_rnn_state }) # Decoder only runs out to max_out; ensure values are identical # to dynamic_rnn, which also zeros out outputs and passes along state. self.assertAllClose( sess_results["final_decoder_outputs"].rnn_output, sess_results["final_rnn_outputs"][:, 0:max_out, :]) if use_sequence_length: self.assertAllClose(sess_results["final_decoder_state"], sess_results["final_rnn_state"])
def _testDynamicDecodeRNN(self, time_major): sequence_length = [3, 4, 3, 1, 0] batch_size = 5 max_time = 8 input_depth = 7 cell_depth = 10 max_out = max(sequence_length) with self.test_session() as sess: if time_major: inputs = np.random.randn(max_time, batch_size, input_depth).astype(np.float32) else: inputs = np.random.randn(batch_size, max_time, input_depth).astype(np.float32) cell = core_rnn_cell.LSTMCell(cell_depth) sampler = sampling_decoder.BasicTrainingSampler( inputs, sequence_length, time_major=time_major) my_decoder = sampling_decoder.BasicSamplingDecoder( cell=cell, sampler=sampler, initial_state=cell.zero_state(dtype=dtypes.float32, batch_size=batch_size)) final_outputs, final_state = decoder.dynamic_decode_rnn( my_decoder, output_time_major=time_major) def _t(shape): if time_major: return (shape[1], shape[0]) + shape[2:] return shape self.assertTrue( isinstance(final_outputs, sampling_decoder.SamplingDecoderOutput)) self.assertTrue( isinstance(final_state, core_rnn_cell.LSTMStateTuple)) self.assertEqual( _t((batch_size, None, cell_depth)), tuple(final_outputs.rnn_output.get_shape().as_list())) self.assertEqual( _t((batch_size, None)), tuple(final_outputs.sample_id.get_shape().as_list())) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "final_outputs": final_outputs, "final_state": final_state }) self.assertEqual(_t((batch_size, max_out, cell_depth)), sess_results["final_outputs"].rnn_output.shape) self.assertEqual(_t((batch_size, max_out)), sess_results["final_outputs"].sample_id.shape)
def _testDynamicDecodeRNN(self, time_major, has_attention): encoder_sequence_length = [3, 2, 3, 1, 0] decoder_sequence_length = [2, 0, 1, 2, 3] batch_size = 5 decoder_max_time = 4 input_depth = 7 cell_depth = 9 attention_depth = 6 vocab_size = 20 end_token = vocab_size - 1 start_token = 0 embedding_dim = 50 max_out = max(decoder_sequence_length) output_layer = layers_core.Dense(vocab_size, use_bias=True, activation=None) beam_width = 3 with self.test_session() as sess: embedding = np.random.randn(vocab_size, embedding_dim).astype(np.float32) cell = core_rnn_cell.LSTMCell(cell_depth) if has_attention: inputs = np.random.randn(batch_size, decoder_max_time, input_depth).astype(np.float32) attention_mechanism = attention_wrapper.BahdanauAttention( num_units=attention_depth, memory=inputs, memory_sequence_length=encoder_sequence_length) cell = attention_wrapper.AttentionWrapper( cell=cell, attention_mechanism=attention_mechanism, attention_size=attention_depth, alignment_history=False) cell_state = cell.zero_state(dtype=dtypes.float32, batch_size=batch_size * beam_width) bsd = beam_search_decoder.BeamSearchDecoder( cell=cell, embedding=embedding, start_tokens=batch_size * [start_token], end_token=end_token, initial_state=cell_state, beam_width=beam_width, output_layer=output_layer, length_penalty_weight=0.0) final_outputs, final_state, final_sequence_lengths = ( decoder.dynamic_decode(bsd, output_time_major=time_major, maximum_iterations=max_out)) def _t(shape): if time_major: return (shape[1], shape[0]) + shape[2:] return shape self.assertTrue( isinstance(final_outputs, beam_search_decoder.FinalBeamSearchDecoderOutput)) self.assertTrue( isinstance(final_state, beam_search_decoder.BeamSearchDecoderState)) beam_search_decoder_output = final_outputs.beam_search_decoder_output self.assertEqual( _t((batch_size, None, beam_width)), tuple(beam_search_decoder_output.scores.get_shape().as_list())) self.assertEqual( _t((batch_size, None, beam_width)), tuple(final_outputs.predicted_ids.get_shape().as_list())) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ 'final_outputs': final_outputs, 'final_state': final_state, 'final_sequence_lengths': final_sequence_lengths }) max_sequence_length = np.max( sess_results['final_sequence_lengths']) # A smoke test self.assertEqual( _t((batch_size, max_sequence_length, beam_width)), sess_results['final_outputs'].beam_search_decoder_output. scores.shape) self.assertEqual( _t((batch_size, max_sequence_length, beam_width)), sess_results['final_outputs'].beam_search_decoder_output. predicted_ids.shape)
def _testStepWithTrainingHelper(self, use_output_layer): sequence_length = [3, 4, 3, 1, 0] batch_size = 5 max_time = 8 input_depth = 7 cell_depth = 10 output_layer_depth = 3 with self.test_session(use_gpu=True) as sess: inputs = np.random.randn(batch_size, max_time, input_depth).astype(np.float32) cell = core_rnn_cell.LSTMCell(cell_depth) helper = helper_py.TrainingHelper(inputs, sequence_length, time_major=False) if use_output_layer: output_layer = layers_core.Dense(output_layer_depth, use_bias=False) expected_output_depth = output_layer_depth else: output_layer = None expected_output_depth = cell_depth my_decoder = basic_decoder.BasicDecoder( cell=cell, helper=helper, initial_state=cell.zero_state(dtype=dtypes.float32, batch_size=batch_size), output_layer=output_layer) output_size = my_decoder.output_size output_dtype = my_decoder.output_dtype self.assertEqual( basic_decoder.BasicDecoderOutput(expected_output_depth, tensor_shape.TensorShape([])), output_size) self.assertEqual( basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), output_dtype) (first_finished, first_inputs, first_state) = my_decoder.initialize() (step_outputs, step_state, step_next_inputs, step_finished) = my_decoder.step(constant_op.constant(0), first_inputs, first_state) batch_size_t = my_decoder.batch_size self.assertTrue( isinstance(first_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) self.assertEqual((batch_size, expected_output_depth), step_outputs[0].get_shape()) self.assertEqual((batch_size, ), step_outputs[1].get_shape()) self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) if use_output_layer: # The output layer was accessed self.assertEqual(len(output_layer.variables), 1) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "batch_size": batch_size_t, "first_finished": first_finished, "first_inputs": first_inputs, "first_state": first_state, "step_outputs": step_outputs, "step_state": step_state, "step_next_inputs": step_next_inputs, "step_finished": step_finished }) self.assertAllEqual([False, False, False, False, True], sess_results["first_finished"]) self.assertAllEqual([False, False, False, True, True], sess_results["step_finished"]) self.assertAllEqual( np.argmax(sess_results["step_outputs"].rnn_output, -1), sess_results["step_outputs"].sample_id)
def _testStepWithScheduledOutputTrainingHelper(self, sampling_probability, use_next_input_layer, use_auxiliary_inputs): sequence_length = [3, 4, 3, 1, 0] batch_size = 5 max_time = 8 input_depth = 7 cell_depth = input_depth if use_next_input_layer: cell_depth = 6 if use_auxiliary_inputs: auxiliary_input_depth = 4 auxiliary_inputs = np.random.randn( batch_size, max_time, auxiliary_input_depth).astype(np.float32) else: auxiliary_inputs = None with self.test_session(use_gpu=True) as sess: inputs = np.random.randn(batch_size, max_time, input_depth).astype(np.float32) cell = core_rnn_cell.LSTMCell(cell_depth) sampling_probability = constant_op.constant(sampling_probability) next_input_layer = None if use_next_input_layer: next_input_layer = layers_core.Dense(input_depth, use_bias=False) helper = helper_py.ScheduledOutputTrainingHelper( inputs=inputs, sequence_length=sequence_length, sampling_probability=sampling_probability, time_major=False, next_input_layer=next_input_layer, auxiliary_inputs=auxiliary_inputs) my_decoder = basic_decoder.BasicDecoder( cell=cell, helper=helper, initial_state=cell.zero_state(dtype=dtypes.float32, batch_size=batch_size)) output_size = my_decoder.output_size output_dtype = my_decoder.output_dtype self.assertEqual( basic_decoder.BasicDecoderOutput(cell_depth, tensor_shape.TensorShape([])), output_size) self.assertEqual( basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), output_dtype) (first_finished, first_inputs, first_state) = my_decoder.initialize() (step_outputs, step_state, step_next_inputs, step_finished) = my_decoder.step(constant_op.constant(0), first_inputs, first_state) if use_next_input_layer: output_after_next_input_layer = next_input_layer( step_outputs.rnn_output) batch_size_t = my_decoder.batch_size self.assertTrue( isinstance(first_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) self.assertEqual((batch_size, ), step_outputs[1].get_shape()) self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) sess.run(variables.global_variables_initializer()) fetches = { "batch_size": batch_size_t, "first_finished": first_finished, "first_inputs": first_inputs, "first_state": first_state, "step_outputs": step_outputs, "step_state": step_state, "step_next_inputs": step_next_inputs, "step_finished": step_finished } if use_next_input_layer: fetches[ "output_after_next_input_layer"] = output_after_next_input_layer sess_results = sess.run(fetches) self.assertAllEqual([False, False, False, False, True], sess_results["first_finished"]) self.assertAllEqual([False, False, False, True, True], sess_results["step_finished"]) sample_ids = sess_results["step_outputs"].sample_id batch_where_not_sampling = np.where(np.logical_not(sample_ids)) batch_where_sampling = np.where(sample_ids) auxiliary_inputs_to_concat = ( auxiliary_inputs[:, 1] if use_auxiliary_inputs else np.array( []).reshape(batch_size, 0).astype(np.float32)) expected_next_sampling_inputs = np.concatenate( (sess_results["output_after_next_input_layer"] [batch_where_sampling] if use_next_input_layer else sess_results["step_outputs"].rnn_output[batch_where_sampling], auxiliary_inputs_to_concat[batch_where_sampling]), axis=-1) self.assertAllClose( sess_results["step_next_inputs"][batch_where_sampling], expected_next_sampling_inputs) self.assertAllClose( sess_results["step_next_inputs"][batch_where_not_sampling], np.concatenate( (np.squeeze(inputs[batch_where_not_sampling, 1], axis=0), auxiliary_inputs_to_concat[batch_where_not_sampling]), axis=-1))
def testStepWithScheduledEmbeddingTrainingHelper(self): sequence_length = [3, 4, 3, 1, 0] batch_size = 5 max_time = 8 input_depth = 7 vocabulary_size = 10 with self.test_session(use_gpu=True) as sess: inputs = np.random.randn(batch_size, max_time, input_depth).astype(np.float32) embeddings = np.random.randn(vocabulary_size, input_depth).astype(np.float32) half = constant_op.constant(0.5) cell = core_rnn_cell.LSTMCell(vocabulary_size) helper = helper_py.ScheduledEmbeddingTrainingHelper( inputs=inputs, sequence_length=sequence_length, embedding=embeddings, sampling_probability=half, time_major=False) my_decoder = basic_decoder.BasicDecoder( cell=cell, helper=helper, initial_state=cell.zero_state(dtype=dtypes.float32, batch_size=batch_size)) output_size = my_decoder.output_size output_dtype = my_decoder.output_dtype self.assertEqual( basic_decoder.BasicDecoderOutput(vocabulary_size, tensor_shape.TensorShape([])), output_size) self.assertEqual( basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), output_dtype) (first_finished, first_inputs, first_state) = my_decoder.initialize() (step_outputs, step_state, step_next_inputs, step_finished) = my_decoder.step(constant_op.constant(0), first_inputs, first_state) batch_size_t = my_decoder.batch_size self.assertTrue( isinstance(first_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) self.assertEqual((batch_size, vocabulary_size), step_outputs[0].get_shape()) self.assertEqual((batch_size, ), step_outputs[1].get_shape()) self.assertEqual((batch_size, vocabulary_size), first_state[0].get_shape()) self.assertEqual((batch_size, vocabulary_size), first_state[1].get_shape()) self.assertEqual((batch_size, vocabulary_size), step_state[0].get_shape()) self.assertEqual((batch_size, vocabulary_size), step_state[1].get_shape()) self.assertEqual((batch_size, input_depth), step_next_inputs.get_shape()) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "batch_size": batch_size_t, "first_finished": first_finished, "first_inputs": first_inputs, "first_state": first_state, "step_outputs": step_outputs, "step_state": step_state, "step_next_inputs": step_next_inputs, "step_finished": step_finished }) self.assertAllEqual([False, False, False, False, True], sess_results["first_finished"]) self.assertAllEqual([False, False, False, True, True], sess_results["step_finished"]) sample_ids = sess_results["step_outputs"].sample_id batch_where_not_sampling = np.where(sample_ids == -1) batch_where_sampling = np.where(sample_ids > -1) self.assertAllClose( sess_results["step_next_inputs"][batch_where_sampling], embeddings[sample_ids[batch_where_sampling]]) self.assertAllClose( sess_results["step_next_inputs"][batch_where_not_sampling], np.squeeze(inputs[batch_where_not_sampling, 1]))
def testStepWithGreedyEmbeddingHelper(self): batch_size = 5 vocabulary_size = 7 cell_depth = vocabulary_size # cell's logits must match vocabulary size input_depth = 10 start_tokens = [0] * batch_size end_token = 1 with self.test_session(use_gpu=True) as sess: embeddings = np.random.randn(vocabulary_size, input_depth).astype(np.float32) cell = core_rnn_cell.LSTMCell(vocabulary_size) helper = helper_py.GreedyEmbeddingHelper(embeddings, start_tokens, end_token) my_decoder = basic_decoder.BasicDecoder( cell=cell, helper=helper, initial_state=cell.zero_state(dtype=dtypes.float32, batch_size=batch_size)) output_size = my_decoder.output_size output_dtype = my_decoder.output_dtype self.assertEqual( basic_decoder.BasicDecoderOutput(cell_depth, tensor_shape.TensorShape([])), output_size) self.assertEqual( basic_decoder.BasicDecoderOutput(dtypes.float32, dtypes.int32), output_dtype) (first_finished, first_inputs, first_state) = my_decoder.initialize() (step_outputs, step_state, step_next_inputs, step_finished) = my_decoder.step(constant_op.constant(0), first_inputs, first_state) batch_size_t = my_decoder.batch_size self.assertTrue( isinstance(first_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_outputs, basic_decoder.BasicDecoderOutput)) self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) self.assertEqual((batch_size, ), step_outputs[1].get_shape()) self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "batch_size": batch_size_t, "first_finished": first_finished, "first_inputs": first_inputs, "first_state": first_state, "step_outputs": step_outputs, "step_state": step_state, "step_next_inputs": step_next_inputs, "step_finished": step_finished }) expected_sample_ids = np.argmax( sess_results["step_outputs"].rnn_output, -1) expected_step_finished = (expected_sample_ids == end_token) expected_step_next_inputs = embeddings[expected_sample_ids] self.assertAllEqual([False, False, False, False, False], sess_results["first_finished"]) self.assertAllEqual(expected_step_finished, sess_results["step_finished"]) self.assertAllEqual(expected_sample_ids, sess_results["step_outputs"].sample_id) self.assertAllEqual(expected_step_next_inputs, sess_results["step_next_inputs"])
def _testDynamicDecodeRNN(self, time_major, maximum_iterations=None): sequence_length = [3, 4, 3, 1, 0] batch_size = 5 max_time = 8 input_depth = 7 cell_depth = 10 max_out = max(sequence_length) with self.test_session(use_gpu=True) as sess: if time_major: inputs = np.random.randn(max_time, batch_size, input_depth).astype(np.float32) else: inputs = np.random.randn(batch_size, max_time, input_depth).astype(np.float32) cell = core_rnn_cell.LSTMCell(cell_depth) helper = helper_py.TrainingHelper(inputs, sequence_length, time_major=time_major) my_decoder = basic_decoder.BasicDecoder( cell=cell, helper=helper, initial_state=cell.zero_state(dtype=dtypes.float32, batch_size=batch_size)) final_outputs, final_state, final_sequence_length = ( decoder.dynamic_decode(my_decoder, output_time_major=time_major, maximum_iterations=maximum_iterations)) def _t(shape): if time_major: return (shape[1], shape[0]) + shape[2:] return shape self.assertTrue( isinstance(final_outputs, basic_decoder.BasicDecoderOutput)) self.assertTrue( isinstance(final_state, core_rnn_cell.LSTMStateTuple)) self.assertEqual( (batch_size, ), tuple(final_sequence_length.get_shape().as_list())) self.assertEqual( _t((batch_size, None, cell_depth)), tuple(final_outputs.rnn_output.get_shape().as_list())) self.assertEqual( _t((batch_size, None)), tuple(final_outputs.sample_id.get_shape().as_list())) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "final_outputs": final_outputs, "final_state": final_state, "final_sequence_length": final_sequence_length, }) # Mostly a smoke test time_steps = max_out if maximum_iterations is not None: time_steps = min(max_out, maximum_iterations) self.assertEqual(_t((batch_size, time_steps, cell_depth)), sess_results["final_outputs"].rnn_output.shape) self.assertEqual(_t((batch_size, time_steps)), sess_results["final_outputs"].sample_id.shape)
def _testWithAttention(self, create_attention_mechanism, expected_final_output, expected_final_state, attention_mechanism_depth=3, alignment_history=False, expected_final_alignment_history=None, name=""): encoder_sequence_length = [3, 2, 3, 1, 0] decoder_sequence_length = [2, 0, 1, 2, 3] batch_size = 5 encoder_max_time = 8 decoder_max_time = 4 input_depth = 7 encoder_output_depth = 10 cell_depth = 9 attention_depth = 6 decoder_inputs = np.random.randn(batch_size, decoder_max_time, input_depth).astype(np.float32) encoder_outputs = np.random.randn(batch_size, encoder_max_time, encoder_output_depth).astype( np.float32) attention_mechanism = create_attention_mechanism( num_units=attention_mechanism_depth, memory=encoder_outputs, memory_sequence_length=encoder_sequence_length) with self.test_session(use_gpu=True) as sess: with vs.variable_scope( "root", initializer=init_ops.random_normal_initializer(stddev=0.01, seed=3)): cell = core_rnn_cell.LSTMCell(cell_depth) cell = wrapper.AttentionWrapper( cell, attention_mechanism, attention_size=attention_depth, alignment_history=alignment_history) helper = helper_py.TrainingHelper(decoder_inputs, decoder_sequence_length) my_decoder = basic_decoder.BasicDecoder( cell=cell, helper=helper, initial_state=cell.zero_state(dtype=dtypes.float32, batch_size=batch_size)) final_outputs, final_state = decoder.dynamic_decode(my_decoder) self.assertTrue( isinstance(final_outputs, basic_decoder.BasicDecoderOutput)) self.assertTrue( isinstance(final_state, wrapper.AttentionWrapperState)) self.assertTrue( isinstance(final_state.cell_state, core_rnn_cell.LSTMStateTuple)) self.assertEqual( (batch_size, None, attention_depth), tuple(final_outputs.rnn_output.get_shape().as_list())) self.assertEqual( (batch_size, None), tuple(final_outputs.sample_id.get_shape().as_list())) self.assertEqual( (batch_size, attention_depth), tuple(final_state.attention.get_shape().as_list())) self.assertEqual( (batch_size, cell_depth), tuple(final_state.cell_state.c.get_shape().as_list())) self.assertEqual( (batch_size, cell_depth), tuple(final_state.cell_state.h.get_shape().as_list())) if alignment_history: state_alignment_history = final_state.alignment_history.stack() # Remove the history from final_state for purposes of the # remainder of the tests. final_state = final_state._replace(alignment_history=()) # pylint: disable=protected-access self.assertEqual( (None, batch_size, encoder_max_time), tuple(state_alignment_history.get_shape().as_list())) else: state_alignment_history = () sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "final_outputs": final_outputs, "final_state": final_state, "state_alignment_history": state_alignment_history, }) print("Copy/paste (%s)\nexpected_final_output = " % name, sess_results["final_outputs"]) sys.stdout.flush() print("Copy/paste (%s)\nexpected_final_state = " % name, sess_results["final_state"]) sys.stdout.flush() print( "Copy/paste (%s)\nexpected_final_alignment_history = " % name, sess_results["state_alignment_history"]) sys.stdout.flush() nest.map_structure(self.assertAllClose, expected_final_output, sess_results["final_outputs"]) nest.map_structure(self.assertAllClose, expected_final_state, sess_results["final_state"]) if alignment_history: # by default, the wrapper emits attention as output self.assertAllClose( # outputs are batch major but the stacked TensorArray is time major sess_results["state_alignment_history"], expected_final_alignment_history)
def _testWithAttention(self, create_attention_mechanism, expected_final_outputs, expected_final_state, attention_mechanism_depth=3): encoder_sequence_length = [3, 2, 3, 1, 0] decoder_sequence_length = [2, 0, 1, 2, 3] batch_size = 5 encoder_max_time = 8 decoder_max_time = 4 input_depth = 7 encoder_output_depth = 10 cell_depth = 9 attention_depth = 6 decoder_inputs = np.random.randn(batch_size, decoder_max_time, input_depth).astype(np.float32) encoder_outputs = np.random.randn(batch_size, encoder_max_time, encoder_output_depth).astype( np.float32) attention_mechanism = create_attention_mechanism( num_units=attention_mechanism_depth, memory=encoder_outputs, memory_sequence_length=encoder_sequence_length) with self.test_session() as sess: with vs.variable_scope( "root", initializer=init_ops.random_normal_initializer(stddev=0.01, seed=3)): cell = core_rnn_cell.LSTMCell(cell_depth) cell = wrapper.DynamicAttentionWrapper( cell, attention_mechanism, attention_size=attention_depth) helper = helper_py.TrainingHelper(decoder_inputs, decoder_sequence_length) my_decoder = basic_decoder.BasicDecoder( cell=cell, helper=helper, initial_state=cell.zero_state(dtype=dtypes.float32, batch_size=batch_size)) final_outputs, final_state = decoder.dynamic_decode(my_decoder) self.assertTrue( isinstance(final_outputs, basic_decoder.BasicDecoderOutput)) self.assertTrue( isinstance(final_state, wrapper.DynamicAttentionWrapperState)) self.assertTrue( isinstance(final_state.cell_state, core_rnn_cell.LSTMStateTuple)) self.assertEqual( (batch_size, None, attention_depth), tuple(final_outputs.rnn_output.get_shape().as_list())) self.assertEqual( (batch_size, None), tuple(final_outputs.sample_id.get_shape().as_list())) self.assertEqual( (batch_size, attention_depth), tuple(final_state.attention.get_shape().as_list())) self.assertEqual( (batch_size, cell_depth), tuple(final_state.cell_state.c.get_shape().as_list())) self.assertEqual( (batch_size, cell_depth), tuple(final_state.cell_state.h.get_shape().as_list())) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "final_outputs": final_outputs, "final_state": final_state }) nest.map_structure(self.assertAllClose, expected_final_outputs, sess_results["final_outputs"]) nest.map_structure(self.assertAllClose, expected_final_state, sess_results["final_state"])
def rnn(hnum1, hnum2, hnum3): file = open(r"wind.csv") file.readline() # 读掉第一行,下次再引用file的时候,将file的文件指针指向第二行开始的文件. reader = csv.reader(file) raw_data = [] for date, hors, u, v, ws, wd in reader: if ws != 'NA': raw_data.append(float(ws)) # difference = max(raw_data)-min(raw_data) # raw_data = [i/difference for i in raw_data] # raw_data=[10*math.sin(0.1*i) for i in range(20000)] sequence_length = 100 # 代表以往数据的移动窗口宽度 注意取值范围 (可调参数) predict_length = 16 # 代表预测数据的移动窗口宽度 即一次预测出的结果数 注意取值范围 (可调参数) train_input_all = [] for i in range(0, len(raw_data[0:-sequence_length - predict_length + 1])): temp_list = [] for j in range(sequence_length): temp_list.append([raw_data[i + j]]) train_input_all.append(temp_list) train_label_all = [] train_label_all1 = [] for i in range(sequence_length, len(raw_data) - predict_length + 1): temp_list = [] for j in range(predict_length): temp_list.append(raw_data[i + j]) train_label_all.append(temp_list) train_label_all1.append(raw_data[i + j]) seperate_point = 5000 # 测试集与训练集分割点 (可调数) test_point = 90000 # 使用的数据量大小(可调数 且必须大于seperate_point) test_point_start = 80000 train_input = train_input_all[0:seperate_point] test_input = train_input_all[test_point_start + 1:test_point] train_output = train_label_all[0:seperate_point] # 训练数据标签格式1 train_output1 = train_label_all1[0:seperate_point] # 训练数据标签格式2 test_output = train_label_all[test_point_start + 1:test_point] # 测试数据标签格式1 test_output1 = train_label_all1[test_point_start + 1:test_point] # 测试数据标签格式2 # 打乱训练集 index = [i for i in range(len(train_input))] shuffle(index) train_input = [train_input[index[i]] for i in range(len(index))] train_output = [train_output[index[i]] for i in range(len(index))] data = tf.placeholder( tf.float32, [None, sequence_length, 1]) # batch_size maxtime deepth target = tf.placeholder(tf.float32, [None, predict_length], name='target') num_hidden = [hnum1, hnum2, hnum3] # 隐含层数量(可调参数) # cell = rnn_cell.BasicRNNCell(num_hidden) # cells = rnn_cell.LSTMCell(num_hidden[0], state_is_tuple=True) cell_layer1 = rnn_cell.LSTMCell(num_hidden[0], state_is_tuple=True) # cell_layer1 = rnn_cell.DropoutWrapper(cell_layer1, input_keep_prob=0.5, output_keep_prob=0.5) cell_layer2 = rnn_cell.LSTMCell(num_hidden[1], state_is_tuple=True) # cell_layer2 = rnn_cell.DropoutWrapper(cell_layer2, input_keep_prob=0.5, output_keep_prob=0.5) cell_layer3 = rnn_cell.LSTMCell(num_hidden[2], state_is_tuple=True) # cell_layer4 = rnn_cell.LSTMCell(num_hidden[3], state_is_tuple=True) # cell_layer5 = rnn_cell.LSTMCell(num_hidden[4], state_is_tuple=True) cells = rnn_cell.MultiRNNCell([cell_layer1, cell_layer2, cell_layer3]) # 建立多层rnn val, state = tf.nn.dynamic_rnn(cells, data, dtype=tf.float32) val = tf.transpose(val, [1, 0, 2]) val_shape = val.get_shape() last = tf.gather(val, int(val.get_shape()[0]) - 1) last_shape = last.get_shape() weight = tf.Variable( tf.truncated_normal([num_hidden[-1], int(target.get_shape()[1])])) bias = tf.Variable(tf.constant(0.1, shape=[target.get_shape()[1]])) prediction = tf.matmul(last, weight) + bias prediction_shape = prediction.get_shape() loss = tf.reduce_mean(tf.square(prediction - target)) loss_shape = loss.get_shape() optimizer = tf.train.AdamOptimizer() minimize = optimizer.minimize(loss) # mistakes = tf.not_equal(tf.argmax(target, 1), tf.argmax(prediction, 1)) error = tf.reduce_mean(tf.square(prediction - target)) error_sep = tf.square(prediction - target) # 计算每一个预测分量的误差 init_op = tf.global_variables_initializer() sess = tf.Session() saver = tf.train.Saver() sess.run(init_op) # 在这里,可以执行这个语句,也可以不执行,即使执行了,初始化的值也会被restore的值给override # saver.restore(sess, r"parameter_5.ckpt") batch_size = 10 # (可调参数) no_of_batches = int(len(train_input) / batch_size) epoch = 25 # (可调参数) total_error1 = 0 predict_result1 = [] total_error = 0 predict_error = [] predict_result = [] temp = 0 # 测试变量 for i in range(epoch): ptr = 0 for j in range(no_of_batches): inp, out = train_input[ptr:ptr + batch_size], train_output[ptr:ptr + batch_size] ptr += batch_size sess.run(minimize, {data: inp, target: out}) print("Epoch - ", str(i)) # sess.run(error, {data: train_input, target: train_output}) # 观察测试样本的估计情况 total_error = 0 predict_error = [] predict_result = [] temp = 0 temp1 = 0 temp_sep = [] total_error_sep = [] for i in range(len(test_input)): inp, out = test_input[i:i + 1], test_output[i:i + 1] temp1 = sess.run(prediction, {data: inp, target: out}) temp = sess.run(error, {data: inp, target: out}) temp_sep = sess.run(error_sep, {data: inp, target: out}) # print(temp1) total_error += temp # predict_error.append(temp) predict_result.append(temp1[0]) total_error_sep.append(temp_sep) total_error /= len(test_input) total_error = math.sqrt(total_error) total_error_sep = (np.array(total_error_sep)).mean(axis=0) # 观察训练样本的训练情况 total_error1 = 0 predict_result1 = [] temp2 = 0 temp3 = 0 temp_sep1 = [] total_error_sep1 = [] for i in range(len(train_input)): inp, out = train_input[i:i + 1], train_output[i:i + 1] temp2 = sess.run(error, {data: inp, target: out}) temp3 = sess.run(prediction, {data: inp, target: out}) temp_sep1 = sess.run(error_sep, {data: inp, target: out}) total_error1 += temp2 predict_result1.append((temp3[0])) total_error_sep1.append(temp_sep1) total_error1 /= len(train_input) total_error1 = math.sqrt(total_error1) total_error_sep1 = (np.array(total_error_sep1)).mean(axis=0) # incorrect = sess.run(error, {data: test_input, target: test_output}) print('Epoch {:2d} error {:3.5f}'.format(i + 1, total_error)) # print('predict_error') # print(predict_error) # print('predict_result') # print(predict_result) saver.save(sess, r"parameter_5.ckpt") sess.close() # saver = tf.train.Saver() # pylab.plot(predict_result)# predict_result1是测试样本的检擦结果predict_result # pylab.plot(test_output) # pylab.plot(predict_result1)# predict_result1是训练样本的检擦结果predict_result # pylab.plot(train_output)#train_output1是训练样本的检查结果test_output1 corrcoef_result_test = [] corrcoef_result_train = [] for cursor in range(16): test_output_single = [ test_output[i][int(cursor)] for i in range(len(test_output)) ] predict_result_single = [ predict_result[i][int(cursor)] for i in range(len(predict_result)) ] corrcoef_result_test.append( corrcoef(test_output_single, predict_result_single)) #print(corrcoef(test_output_single, predict_result_single)) for cursor in range(16): train_output_single = [ train_output[i][int(cursor)] for i in range(len(train_output)) ] predict_result1_single = [ predict_result1[i][int(cursor)] for i in range(len(predict_result1)) ] corrcoef_result_train.append( corrcoef(train_output_single, predict_result1_single)) #print(corrcoef(train_output_single, predict_result1_single)) ''' 需要记录的数据 1输入训练样本编号 2测试样本编号 3使用的模型类型 4使用的模型参数(隐含层数量 隐含层每层的单元数量)5训练batch大小 6训练的周期数 7预测结果 8输出误差大小 9输出的相关系数 10训练时间 ''' csvfile = open(r'short_result5.csv', 'a') writer = csv.writer(csvfile) writer.writerow([ 'epoch', 'seperate_point', 'test_point_start', 'test_point', 'num_hidden', 'sequence_length', 'predict_length', 'batch_size' ]) data = [(epoch, seperate_point, test_point_start, test_point, num_hidden, sequence_length, predict_length, batch_size)] writer.writerows(data) writer.writerow(['corrcoef_result_test']) writer.writerow(corrcoef_result_test) writer.writerow(['corrcoef_result_train']) writer.writerow(corrcoef_result_train) writer.writerow(['prediction_result']) writer.writerow(['total_error_sep']) for i in range(len(total_error_sep)): writer.writerow(total_error_sep[i]) writer.writerow(['total_error']) writer.writerow([total_error]) csvfile.close()
def testStepWithBasicTrainingSampler(self): sequence_length = [3, 4, 3, 1, 0] batch_size = 5 max_time = 8 input_depth = 7 cell_depth = 10 with self.test_session() as sess: inputs = np.random.randn(batch_size, max_time, input_depth).astype(np.float32) cell = core_rnn_cell.LSTMCell(cell_depth) sampler = sampling_decoder.BasicTrainingSampler( inputs, sequence_length, time_major=False) my_decoder = sampling_decoder.BasicSamplingDecoder( cell=cell, sampler=sampler, initial_state=cell.zero_state( dtype=dtypes.float32, batch_size=batch_size)) output_size = my_decoder.output_size output_dtype = my_decoder.output_dtype self.assertEqual( sampling_decoder.SamplingDecoderOutput(cell_depth, tensor_shape.TensorShape([])), output_size) self.assertEqual( sampling_decoder.SamplingDecoderOutput(dtypes.float32, dtypes.int32), output_dtype) (first_finished, first_inputs, first_state) = my_decoder.initialize() (step_outputs, step_state, step_next_inputs, step_finished) = my_decoder.step( constant_op.constant(0), first_inputs, first_state) batch_size_t = my_decoder.batch_size self.assertTrue(isinstance(first_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue(isinstance(step_state, core_rnn_cell.LSTMStateTuple)) self.assertTrue( isinstance(step_outputs, sampling_decoder.SamplingDecoderOutput)) self.assertEqual((batch_size, cell_depth), step_outputs[0].get_shape()) self.assertEqual((batch_size,), step_outputs[1].get_shape()) self.assertEqual((batch_size, cell_depth), first_state[0].get_shape()) self.assertEqual((batch_size, cell_depth), first_state[1].get_shape()) self.assertEqual((batch_size, cell_depth), step_state[0].get_shape()) self.assertEqual((batch_size, cell_depth), step_state[1].get_shape()) sess.run(variables.global_variables_initializer()) sess_results = sess.run({ "batch_size": batch_size_t, "first_finished": first_finished, "first_inputs": first_inputs, "first_state": first_state, "step_outputs": step_outputs, "step_state": step_state, "step_next_inputs": step_next_inputs, "step_finished": step_finished }) self.assertAllEqual([False, False, False, False, True], sess_results["first_finished"]) self.assertAllEqual([False, False, False, True, True], sess_results["step_finished"]) self.assertAllEqual( np.argmax(sess_results["step_outputs"].rnn_output, -1), sess_results["step_outputs"].sample_id)
''' index = [i for i in range(len(test_input))] shuffle(index) test_input = [test_input[index[i]] for i in range(len(index))] test_output = [test_output[index[i]] for i in range(len(index))] ''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' ''''' ''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '''''' '' data = tf.placeholder(tf.float32, [None, sequence_length, 1]) #batch_size maxtime deepth target = tf.placeholder(tf.float32, [None, predict_length], name='target') num_hidden = [30, 30] #隐含层数量(可调参数) #cell = rnn_cell.BasicRNNCell(num_hidden) #cells = rnn_cell.LSTMCell(num_hidden[0], state_is_tuple=True) cell_layer1 = rnn_cell.LSTMCell(num_hidden[0], state_is_tuple=True) #cell_layer1 = rnn_cell.DropoutWrapper(cell_layer1, input_keep_prob=0.5, output_keep_prob=0.5) cell_layer2 = rnn_cell.LSTMCell(num_hidden[1], state_is_tuple=True) #cell_layer2 = rnn_cell.DropoutWrapper(cell_layer2, input_keep_prob=0.5, output_keep_prob=0.5) #cell_layer3 = rnn_cell.LSTMCell(num_hidden[2], state_is_tuple=True) #cell_layer4 = rnn_cell.LSTMCell(num_hidden[3], state_is_tuple=True) #cell_layer5 = rnn_cell.LSTMCell(num_hidden[4], state_is_tuple=True) cells = rnn_cell.MultiRNNCell([cell_layer1, cell_layer2]) #建立多层rnn val, state = tf.nn.dynamic_rnn(cells, data, dtype=tf.float32) val = tf.transpose(val, [1, 0, 2]) val_shape = val.get_shape() last = tf.gather(val, int(val.get_shape()[0]) - 1)
def __init__(self, name, env_state_size, env_stack_size, num_actions, hidden_size, num_stacks, batch_size, scope): self.name = name self.batch_size = batch_size self.hidden_size = hidden_size self.lstm = core_rnn_cell.LSTMCell(num_units=hidden_size) self.zero_state = self.lstm.zero_state(batch_size=self.batch_size, dtype=tf.float32) self.rnn_state_ph = tf.placeholder(shape=[2, batch_size, hidden_size], dtype=tf.float32) self.rnn_state = self.to_lstm_state(self.rnn_state_ph) self.env_state_input = tf.placeholder( shape=[batch_size, None, env_state_size], dtype=tf.float32) self.env_stack_input = tf.placeholder(shape=[ batch_size, None, num_stacks * env_stack_size * env_state_size ], dtype=tf.float32) self.action_softmax_div = tf.placeholder(shape=[None, 1], dtype=tf.float32) self.stack_softmax_div = tf.placeholder(shape=[None, 1], dtype=tf.float32) self.env_input = tf.concat( (self.env_state_input, self.env_stack_input), axis=2) hidden, self.hidden_state = rnn.dynamic_rnn( self.lstm, self.env_input, initial_state=self.rnn_state) out_act = slim.fully_connected(hidden, num_actions, activation_fn=None, biases_initializer=None) self.output = tf.nn.softmax(out_act / self.action_softmax_div) self.chosen_action = tf.argmax(self.output, axis=2) out_sta = slim.fully_connected(hidden, num_stacks, activation_fn=None, biases_initializer=None) self.out_stack = tf.nn.softmax(out_sta / self.stack_softmax_div) self.chosen_stack = tf.argmax(self.out_stack, axis=2) # The next six lines establish the training procedure. We feed the reward and chosen action into the network # to compute the loss, and use it to update the network. self.action_reward_holder = tf.placeholder(shape=[batch_size, None], dtype=tf.float32) self.stack_reward_holder = tf.placeholder(shape=[batch_size, None], dtype=tf.float32) self.action_holder = tf.placeholder(shape=[batch_size, None], dtype=tf.int32) self.stack_holder = tf.placeholder(shape=[batch_size, None], dtype=tf.int32) loss_actions, loss_stacks = tf.scan( self.determine_input_seq, elems=[ self.output, self.action_holder, self.out_stack, self.stack_holder, self.action_reward_holder, self.stack_reward_holder ], initializer=(tf.constant(0.0), tf.constant(0.0))) self.loss = loss_actions + loss_stacks self.trainable_variables = tf.get_collection( tf.GraphKeys.TRAINABLE_VARIABLES, scope=scope.name) self.gradients = tf.gradients(self.loss, self.trainable_variables) optimizer = tf.train.AdamOptimizer() self.update = optimizer.apply_gradients( zip(self.gradients, self.trainable_variables)) self.trained_variables = [] self._saver = tf.train.Saver(self.trainable_variables)