def __init__(self, max_len, input_size, size, num_layers,
max_gradient_norm, batch_size, learning_rate,
learning_rate_decay_factor):
"""Create the network. A simplified network that handles only sorting.
Args:
max_len: maximum length of the model.
input_size: size of the inputs data.
size: number of units in each layer of the model.
num_layers: number of layers in the model.
max_gradient_norm: gradients will be clipped to maximally this norm.
batch_size: the size of the batches used during training;
the model construction is independent of batch_size, so it can be
changed after initialization if this is convenient, e.g., for decoding.
learning_rate: learning rate to start with.
learning_rate_decay_factor: decay learning rate by this much when needed.
"""
self.batch_size = batch_size
self.learning_rate = tf.Variable(float(learning_rate), trainable=False)
self.learning_rate_decay_op = self.learning_rate.assign(
self.learning_rate * learning_rate_decay_factor)
self.global_step = tf.Variable(0, trainable=False)
cell = tf.nn.rnn_cell.GRUCell(size)
if num_layers > 1:
cell = tf.nn.rnn_cell.MultiRNNCell([single_cell] * num_layers)
self.encoder_inputs = []
self.decoder_inputs = []
self.decoder_targets = []
self.target_weights = []
for i in range(max_len):
self.encoder_inputs.append(
tf.placeholder(tf.float32, [batch_size, input_size],
name="EncoderInput%d" % i))
for i in range(max_len + 1):
self.decoder_inputs.append(
tf.placeholder(tf.float32, [batch_size, input_size],
name="DecoderInput%d" % i))
self.decoder_targets.append(
tf.placeholder(tf.float32, [batch_size, max_len + 1],
name="DecoderTarget%d" % i)) # one hot
self.target_weights.append(
tf.placeholder(tf.float32, [batch_size, 1],
name="TargetWeight%d" % i))
# Encoder
# Need for attention
encoder_outputs, final_state = tf.nn.rnn(cell,
self.encoder_inputs,
dtype=tf.float32)
# Need a dummy output to point on it. End of decoding.
encoder_outputs = [tf.zeros([FLAGS.batch_size, FLAGS.rnn_size])
] + encoder_outputs
# First calculate a concatenation of encoder outputs to put attention on.
top_states = [
tf.reshape(e, [-1, 1, cell.output_size]) for e in encoder_outputs
]
attention_states = tf.concat(1, top_states)
with tf.variable_scope("decoder"):
outputs, states, _ = pointer_decoder(self.decoder_inputs,
final_state, attention_states,
cell)
with tf.variable_scope("decoder", reuse=True):
predictions, _, inps = pointer_decoder(self.decoder_inputs,
final_state,
attention_states,
cell,
feed_prev=True)
self.predictions = predictions
self.outputs = outputs
self.inps = inps
def __init__(self, max_len, input_size, size, num_layers, max_gradient_norm,
batch_size, learning_rate, learning_rate_decay_factor):
"""Create the network. A simplified network that handles only sorting.
Args:
max_len: maximum length of the model.
input_size: size of the inputs data.
size: number of units in each layer of the model.
num_layers: number of layers in the model.
max_gradient_norm: gradients will be clipped to maximally this norm.
batch_size: the size of the batches used during training;
the model construction is independent of batch_size, so it can be
changed after initialization if this is convenient, e.g., for decoding.
learning_rate: learning rate to start with.
learning_rate_decay_factor: decay learning rate by this much when needed.
"""
self.batch_size = batch_size
self.learning_rate = tf.Variable(float(learning_rate), trainable=False)
self.learning_rate_decay_op = self.learning_rate.assign(
self.learning_rate * learning_rate_decay_factor)
self.global_step = tf.Variable(0, trainable=False)
cell = rnn_cell.GRUCell(size)
if num_layers > 1:
cell = rnn_cell.MultiRNNCell([single_cell] * num_layers)
self.encoder_inputs = []
self.decoder_inputs = []
self.decoder_targets = []
self.target_weights = []
for i in range(max_len):
self.encoder_inputs.append(tf.placeholder(
tf.float32, [batch_size, input_size], name="EncoderInput%d" % i))
for i in range(max_len + 1):
self.decoder_inputs.append(tf.placeholder(
tf.float32, [batch_size, input_size], name="DecoderInput%d" % i))
self.decoder_targets.append(tf.placeholder(
tf.float32, [batch_size, max_len + 1], name="DecoderTarget%d" % i)) # one hot
self.target_weights.append(tf.placeholder(
tf.float32, [batch_size, 1], name="TargetWeight%d" % i))
# Encoder
# Need for attention
encoder_outputs, final_state = rnn.rnn(cell, self.encoder_inputs, dtype=tf.float32)
# Need a dummy output to point on it. End of decoding.
encoder_outputs = [tf.zeros([FLAGS.batch_size, FLAGS.rnn_size])] + encoder_outputs
# First calculate a concatenation of encoder outputs to put attention on.
top_states = [tf.reshape(e, [-1, 1, cell.output_size])
for e in encoder_outputs]
attention_states = tf.concat(1, top_states)
with tf.variable_scope("decoder"):
outputs, states, _ = pointer_decoder(
self.decoder_inputs, final_state, attention_states, cell)
with tf.variable_scope("decoder", reuse=True):
predictions, _, inps = pointer_decoder(
self.decoder_inputs, final_state, attention_states, cell, feed_prev=True)
self.predictions = predictions
self.outputs = outputs
self.inps = inps
def __init__(self, max_len, input_size, size, num_layers, batch_size,
learning_rate):
"""Create the network.
Args:
max_len: maximum length of the model.
input_size: size of the inputs data.
size: number of units in each layer of the model.
num_layers: number of layers in the model.
batch_size: the size of the batches used during training;
the model construction is independent of batch_size, so it can be
changed after initialization if this is convenient, e.g., for decoding.
learning_rate: learning rate to start with.
"""
self.max_len = max_len
self.batch_size = batch_size
self.learning_rate = learning_rate
self.global_step = tf.Variable(0, trainable=False)
cell = tf.nn.rnn_cell.LSTMCell(
size, initializer=tf.random_uniform_initializer(-0.08, 0.08))
if num_layers > 1:
cell = tf.nn.rnn_cell.MultiRNNCell([cell] * num_layers)
self.encoder_inputs = []
self.decoder_inputs = []
self.decoder_targets = []
self.target_weights = []
for i in range(max_len):
self.encoder_inputs.append(
tf.placeholder(tf.float32, [batch_size, input_size],
name="EncoderInput%d" % i))
for i in range(max_len + 1):
self.decoder_inputs.append(
tf.placeholder(tf.float32, [batch_size, input_size],
name="DecoderInput%d" % i))
self.decoder_targets.append(
tf.placeholder(tf.float32, [batch_size, max_len + 1],
name="DecoderTarget%d" % i)) # one hot
self.target_weights.append(
tf.placeholder(tf.float32, [batch_size, 1],
name="TargetWeight%d" % i))
# Need for attention
encoder_outputs, final_state = tf.nn.rnn(cell,
self.encoder_inputs,
dtype=tf.float32)
# Need a dummy output to point on it. End of decoding.
encoder_outputs = [tf.zeros([batch_size, size])] + encoder_outputs
# First calculate a concatenation of encoder outputs to put attention on.
top_states = [
tf.reshape(e, [-1, 1, cell.output_size]) for e in encoder_outputs
]
attention_states = tf.concat(1, top_states)
#For training
with tf.variable_scope("decoder"):
outputs, states, _ = pointer_decoder(
self.decoder_inputs,
final_state,
attention_states,
cell,
feed_prev=False,
pointer_type=FLAGS.pointer_type)
#For inference
with tf.variable_scope("decoder", reuse=True):
predictions, _, inps = pointer_decoder(
self.decoder_inputs,
final_state,
attention_states,
cell,
feed_prev=True,
pointer_type=FLAGS.pointer_type)
self.predictions = predictions
self.outputs = outputs
self.inps = inps
def __init__(self, max_len, input_size, size, num_layers,
max_gradient_norm, batch_size, learning_rate,
learning_rate_decay_factor, inter_dim, fc_dim, use_cnn,
resnet_cnn, image_dim, vgg_dim, bidirect):
"""Create the network. A simplified network that handles only sorting.
Args:
max_len: maximum length of the model.
input_size: size of the inputs data.
size: number of units in each layer of the model.
num_layers: number of layers in the model.
max_gradient_norm: gradients will be clipped to maximally this norm.
batch_size: the size of the batches used during training;
the model construction is independent of batch_size, so it can be
changed after initialization if this is convenient, e.g., for decoding.
learning_rate: learning rate to start with.
learning_rate_decay_factor: decay learning rate by this much when needed.
"""
self.init = tf.contrib.layers.variance_scaling_initializer()
self.batch_size = batch_size
self.learning_rate = tf.Variable(float(learning_rate), trainable=False)
self.learning_rate_decay_op = self.learning_rate.assign(
self.learning_rate * learning_rate_decay_factor)
self.global_step = tf.Variable(0, trainable=False)
self.keep_prob = tf.placeholder(tf.float32, name="dropout_pbty")
cell, cell_fw, cell_bw = None, None, None
# https://github.com/devsisters/pointer-network-tensorflow/blob/master/model.py#L148
if num_layers == 1:
if bidirect:
cell_fw, cell_bw = self.getCell(size), self.getCell(size)
decoder_cell = self.getCell(size * 2)
else:
cell = self.getCell(size)
decoder_cell = self.getCell(size)
elif num_layers > 1:
if bidirect:
cell_fw = tf.contrib.rnn.MultiRNNCell(
[self.getCell(size) for _ in range(num_layers)],
state_is_tuple=True)
cell_bw = tf.contrib.rnn.MultiRNNCell(
[self.getCell(size) for _ in range(num_layers)],
state_is_tuple=True)
decoder_cell = tf.contrib.rnn.MultiRNNCell(
[self.getCell(size * 2) for _ in range(num_layers)],
state_is_tuple=True)
else:
cell = tf.contrib.rnn.MultiRNNCell(
[self.getCell(size) for _ in range(num_layers)],
state_is_tuple=True)
decoder_cell = tf.contrib.rnn.MultiRNNCell(
[self.getCell(size) for _ in range(num_layers)],
state_is_tuple=True)
self.encoder_inputs = []
self.decoder_inputs = []
self.decoder_targets = []
self.target_weights = []
for i in range(max_len):
i_size = [batch_size, input_size] if not use_cnn else [
batch_size, image_dim, image_dim, 3
]
self.encoder_inputs.append(
tf.placeholder(tf.float32, i_size, name="EncoderInput%d" % i))
for i in range(max_len + 1):
i_size = [batch_size, input_size] if not use_cnn else [
batch_size, image_dim, image_dim, 3
]
if i > 0:
self.decoder_inputs.append(
tf.placeholder(tf.float32,
i_size,
name="DecoderInput%d" % i))
self.decoder_targets.append(
tf.placeholder(tf.float32, [batch_size, max_len + 1],
name="DecoderTarget%d" % i)) # one hot
self.target_weights.append(
tf.placeholder(tf.float32, [batch_size, 1],
name="TargetWeight%d" % i))
# Encoder
trainable_init_state = tf.Variable(tf.zeros(i_size),
"trainable_init_state")
self.decoder_inputs_updated = [trainable_init_state
] + self.decoder_inputs
# Neeed to pass both encode inputs and everything through a dense layer.
if use_cnn:
# Encoder
if resnet_cnn:
cnn_f_extractor = cnn_resnet.CNN_FeatureExtractor()
else:
cnn_f_extractor = cnn.CNN_FeatureExtractor()
all_inps = []
num_encoder = len(self.encoder_inputs)
all_inps.extend(self.encoder_inputs)
all_inps.extend(self.decoder_inputs_updated)
stacked_ins = tf.stack(all_inps)
stacked_ins = tf.reshape(stacked_ins,
[-1, image_dim, image_dim, 3])
if resnet_cnn:
inputfn, features = cnn_f_extractor.getCNNFeatures(
stacked_ins, fc_dim, self.init)
else:
inputfn, features = cnn_f_extractor.getCNNFeatures(
stacked_ins, vgg_dim, fc_dim, self.init)
self.print_out = features[0]
self.inputfn = inputfn
features = tf.reshape(features, [-1, batch_size, fc_dim])
all_out = tf.unstack(features)
self.proj_encoder_inputs = all_out[:num_encoder]
self.proj_decoder_inputs = all_out[num_encoder:]
else:
with vs.variable_scope("projector_scope"):
W1 = vs.get_variable("W1", [input_size, inter_dim])
b1 = vs.get_variable("b1", [inter_dim])
W2 = vs.get_variable("W2", [inter_dim, fc_dim])
b2 = vs.get_variable("b2", [fc_dim])
self.proj_encoder_inputs = []
for inp in self.encoder_inputs:
out = tf.nn.relu(tf.matmul(inp, W1) + b1)
out = tf.nn.relu(tf.matmul(out, W2) + b2)
self.proj_encoder_inputs.append(out)
with vs.variable_scope("projector_scope", reuse=True):
W1 = vs.get_variable("W1", [input_size, inter_dim])
b1 = vs.get_variable("b1", [inter_dim])
W2 = vs.get_variable("W2", [inter_dim, fc_dim])
b2 = vs.get_variable("b2", [fc_dim])
self.proj_decoder_inputs = []
for inp in self.decoder_inputs:
out = tf.nn.relu(tf.matmul(inp, W1) + b1)
out = tf.nn.relu(tf.matmul(out, W2) + b2)
self.proj_decoder_inputs.append(out)
# Need for attention
if not bidirect:
encoder_outputs, final_state = tf.contrib.rnn.static_rnn(
cell, self.proj_encoder_inputs, dtype=tf.float32)
#if num_layers > 1 : final_state = final_state[-1]
else:
encoder_outputs, output_state_fw, output_state_bw = tf.contrib.rnn.static_bidirectional_rnn(
cell_fw, cell_bw, self.proj_encoder_inputs, dtype=tf.float32)
if num_layers > 1:
final_state = []
for ind, fw in enumerate(output_state_fw):
bw = output_state_bw[ind]
final_state.append(tf.concat([fw, bw], axis=1))
#output_state_fw, output_state_bw = output_state_fw[-1], output_state_bw[-1]
if FLAGS.cell_type == "LSTM": # DOESN'T WORK FOR NUM_LAYERS > 1
if num_layers == 1:
output_state_fw, output_state_bw = [output_state_fw
], [output_state_bw]
final_state = []
for ind, fw in enumerate(output_state_fw):
bw = output_state_bw[ind]
o_fw_c, o_fw_m = tf.unstack(fw, axis=0)
o_bw_c, o_bw_m = tf.unstack(bw, axis=0)
o_c = tf.concat([o_fw_c, o_bw_c], axis=1)
o_m = tf.concat([o_fw_m, o_bw_m], axis=1)
final_state.append(tf.contrib.rnn.LSTMStateTuple(o_c, o_m))
if num_layers == 1: final_state = final_state[0]
elif num_layers <= 1:
final_state = tf.concat([output_state_fw, output_state_bw],
axis=1)
output_size = size if not bidirect else 2 * size
#output_size = output_size*num_layers
# Need a dummy output to point on it. End of decoding.
encoder_outputs = [
tf.zeros([FLAGS.batch_size, output_size])
] + encoder_outputs #[tf.zeros([FLAGS.batch_size, output_size])] + encoder_outputs
# First calculate a concatenation of encoder outputs to put attention on.
#print(encoder_outputs[1].get_shape(), output_state_fw.get_shape(), output_size)
top_states = [
tf.reshape(e, [-1, 1, output_size]) for e in encoder_outputs
]
attention_states = tf.concat(axis=1, values=top_states)
with tf.variable_scope("decoder"):
outputs, states, _, _ = pointer_decoder(
self.proj_decoder_inputs,
final_state,
attention_states,
decoder_cell,
cell_type=FLAGS.cell_type,
num_glimpses=FLAGS.num_glimpses,
num_layers=num_layers)
#print("DECODING")
with tf.variable_scope("decoder", reuse=True):
predictions, _, inps, cp = pointer_decoder(
self.proj_decoder_inputs,
final_state,
attention_states,
decoder_cell,
feed_prev=True,
one_hot=FLAGS.encoder_attn_1hot,
cell_type=FLAGS.cell_type,
num_glimpses=FLAGS.num_glimpses,
num_layers=num_layers)
self.predictions = predictions
#self.cps = tf.transpose(tf.stack(cp), (1, 0))
self.outputs = outputs
self.inps = inps
def __init__(self,max_len,input_size,size,num_layers,max_gradient_norm,batch_size,learning_rate,learning_reate_decay_factor):
'''
Create a simple network that handles only sorting
:param max_len: maximum length of model
:param input_size:size of input data
:param size:number of units in each layer in the model
:param num_layers:number of layers in the model
:param max_gradient_norm:gradients will be clipped to maximally this norm.
:param batch_size: the size of the batches used during training;
the model construction is independent of batch_size, so it can be
changed after initialization if this is convenient, e.g., for decoding.
:param learning_rate:learning rate to start with.
:param learning_reate_decay_factor:decay learning rate by this much when needed.
'''
self.batch_size=batch_size
self.learning_rate=tf.Variable(float(learning_rate),trainable=True)
self.learning_rate_decay_op=self.learning_rate.assign(
self.learning_rate*learning_reate_decay_factor
)
self.global_step=tf.Variable(0,trainable=False)
# with tf.variable_scope(name_or_scope='cell',reuse=tf.AUTO_REUSE):
# if num_layers>1:
with tf.variable_scope('cell',reuse=tf.AUTO_REUSE):
#cell=tf.contrib.rnn.GRUCell(size)
cell=tf.nn.rnn_cell.BasicRNNCell(size)
#注意：这里一旦num_layers不为1 就报维度错误，具体原因不清楚
if num_layers>1:
cell=tf.contrib.rnn.MultiRNNCell([cell for _ in range(num_layers)])
# init_state = cell.zero_state(batch_size, dtype=tf.float32)
#cell=tf.nn.rnn_cell.MultiRNNCell([self.get_cell(size)],state_is_tuple=True)
# else:
# cell=self.get_cell(size)
self.encoder_inputs=[]
self.decoder_inputs=[]
self.decoder_targets=[]
# weights后面传入的值其实全为1，并没有太大的意义
self.target_weights=[]
# 这样写的目的是为了可以输入到RNN模型中，而不用再次执行split操作
for i in range(max_len):
self.encoder_inputs.append(tf.placeholder(
tf.float32,[batch_size,input_size],name='EncoderInput%d'%i
))
for i in range(max_len+1):
self.decoder_inputs.append(tf.placeholder(
tf.float32,[batch_size,input_size],name='DecoderInput%i'%i))
self.decoder_targets.append(tf.placeholder(
tf.float32,[batch_size,max_len+1],name='DecoderTarget%d'%i)) # one hot
self.target_weights.append(tf.placeholder(
tf.float32,[batch_size,1],name='TargetWeight%d'%i))
# Encoder
# Need for attention
encoder_outputs,final_state=tf.contrib.rnn.static_rnn(cell,self.encoder_inputs,dtype=tf.float32)
# Need a dummy ouput to point on it. End of decoding
# 相当与在0位置添加了一种选择，当选择到了0位置以后，即可结束decoder部分
encoder_outputs=[tf.zeros((FLAGS.batch_size,FLAGS.rnn_size))]+encoder_outputs
print('encoder_ouputs:',encoder_outputs)
# First calculate a concatenation of encoder outputs to put attention on
top_state=[tf.reshape(e,[-1,1,cell.output_size]) for e in encoder_outputs]
# attention_states:shape:(batch_size,max_len+1,rnn_size)
attention_states=tf.concat(values=top_state,axis=1)
with tf.variable_scope('decoder'):
outputs,states,_=pointer_decoder(
self.decoder_inputs,final_state,attention_states,cell)
# 这个代码测试写的比较简单，用的就是训练集做为测试
with tf.variable_scope('decoder',reuse=True):
predictions, _, inps = pointer_decoder(
self.decoder_inputs, final_state, attention_states, cell, feed_prev=True)
# predictions:list,每个元素为（30，11）
#final state:(30,32)
self.predictions=predictions
self.outputs=outputs
self.inps=inps