def _convLSTM(self,
input_hidden_state,
scope_name='convLSTM',
initial_state=None,
trainable=True,
scope_reuse=False):
with tf.variable_scope(scope_name) as scope:
if scope_reuse:
scope.reuse_variables()
# Create a placeholder for videos.
print scope_name, input_hidden_state.get_shape()
cell = ConvLSTMCell([self.IMAGE_HEIGHT / 8, self.IMAGE_WIDTH / 8],
self.lstm_channel, self.conLSTM_kernel)
if initial_state == None:
outputs, state = tf.nn.dynamic_rnn(
cell,
input_hidden_state,
initial_state=cell.zero_state(1, dtype=tf.float32),
dtype=input_hidden_state.dtype)
else:
outputs, state = tf.nn.dynamic_rnn(
cell,
input_hidden_state,
initial_state=initial_state,
dtype=input_hidden_state.dtype)
print scope_name, outputs.get_shape()
return outputs, state, cell.return_weight()
def conv_lstm_encoder(self, H, W, filter_size, kernel, encoder_input):
with tf.variable_scope('enc_lstm_model',
reuse=self.reuse_conv_lstm_encoder):
encoder_cell = ConvLSTMCell([H, W],
filter_size,
kernel,
reuse=tf.get_variable_scope().reuse)
zero_state = encoder_cell.zero_state(self.batch_size,
dtype=tf.float32)
_, encoded_state = tf.nn.dynamic_rnn(cell=encoder_cell,
inputs=encoder_input,
initial_state=zero_state)
self.reuse_conv_lstm_encoder = True
return encoded_state
def feedforward(self, ml, mh, regularizer=0):
input_shape = mh.get_shape().as_list()
batch_size = input_shape[0]
#LSTM
with tf.variable_scope('LSTM_1'):
rh = tf.get_variable(
"rh",
[batch_size, self.output_h, self.output_w, self.output_depth],
initializer=tf.constant_initializer(0.0),
trainable=False)
conv_w1 = tf.get_variable(
"weight_lstm1",
filter_shape,
initializer=tf.truncated_normal_initializer(stddev=0.001))
conv_b1 = tf.get_variable("bias1",
1,
initializer=tf.constant_initializer(0.0))
cell_g1 = ConvLSTMCell([self.output_h, self.output_w],
filters_lstm, kernal_lstm)
init_state_g1 = cell_g1.zero_state(batch_size, dtype=tf.float32)
state1 = init_state_g1
for timestep in range(iteration):
if timestep > 0:
tf.get_variable_scope().reuse_variables()
rh_tr = tf.transpose(rh, perm=[0, 2, 1, 3])
g1 = odl_op_layer_adjoint((odl_op_layer(rh_tr) - mh))
gt1 = tf.transpose(g1, perm=[0, 2, 1, 3])
(cell_output1, state1) = cell_g1(gt1, state1)
conv1 = tf.nn.conv2d(cell_output1, conv_w1, [1, 1, 1, 1],
"VALID")
s1 = tf.nn.tanh(tf.nn.bias_add(conv1, conv_b1))
self.variable_summaries(s1, ('s1_%d' % timestep))
rh = rh + 0.0001 * s1 * gt1
rh = tf.clip_by_value(rh, 0, 5)
tf.summary.image('rh_pred_%d' % timestep, rh, 1)
with tf.variable_scope('LSTM_2'):
rl = tf.get_variable(
"rl",
[batch_size, self.output_h, self.output_w, self.output_depth],
initializer=tf.constant_initializer(0.0),
trainable=False)
conv_w2 = tf.get_variable(
"weight_lstm2",
filter_shape,
initializer=tf.truncated_normal_initializer(stddev=0.001))
conv_b2 = tf.get_variable("bias2",
1,
initializer=tf.constant_initializer(0.0))
cell_g2 = ConvLSTMCell([self.output_h, self.output_w],
filters_lstm, kernal_lstm)
init_state_g2 = cell_g2.zero_state(batch_size, dtype=tf.float32)
state2 = init_state_g2
for timestep in range(iteration):
if timestep > 0:
tf.get_variable_scope().reuse_variables()
rl_tr = tf.transpose(rl, perm=[0, 2, 1, 3])
g2 = odl_op_layer_adjoint((odl_op_layer(rl_tr) - ml))
gt2 = tf.transpose(g2, perm=[0, 2, 1, 3])
(cell_output2, state2) = cell_g2(gt2, state2)
conv2 = tf.nn.conv2d(cell_output2, conv_w2, [1, 1, 1, 1],
"VALID")
s2 = tf.nn.tanh(tf.nn.bias_add(conv2, conv_b2))
self.variable_summaries(s2, ('s2_%d' % timestep))
rl = rl + 0.0001 * s2 * gt2
rl = tf.clip_by_value(rl, 0, 5)
tf.summary.image('rl_pred_%d' % timestep, rl, 1)
#CNN
layer_L1 = layer_xyf.convo(rl, "conv_L1", FILTER_1, STRIDE_1, PAD_1)
layer_L2 = layer_xyf.convo(layer_L1, "conv_L2", FILTER_2, STRIDE_2,
PAD_2)
layer_L3 = layer_xyf.convo(layer_L2, "conv_L3", FILTER_3, STRIDE_3,
PAD_3)
layer_L4 = layer_xyf.convo(layer_L3, "conv_L4", FILTER_4, STRIDE_4,
PAD_4)
layer_L5 = layer_xyf.convo(layer_L4, "conv_L5", FILTER_5, STRIDE_5,
PAD_5)
layer_H1 = layer_xyf.convo(rh, "conv_H1", FILTER_1, STRIDE_1, PAD_1)
layer_H2 = layer_xyf.convo(layer_H1, "conv_H2", FILTER_2, STRIDE_2,
PAD_2)
layer_H3 = layer_xyf.convo(layer_H2, "conv_H3", FILTER_3, STRIDE_3,
PAD_3)
layer_H4 = layer_xyf.convo(layer_H3, "conv_H4", FILTER_4, STRIDE_4,
PAD_4)
layer_H5 = layer_xyf.convo(layer_H4, "conv_H5", FILTER_5, STRIDE_5,
PAD_5)
combine_LH = tf.concat([layer_L5, layer_H5], 3)
pa_pred = layer_xyf.convo_noneRelu(combine_LH, "conv_pa", FILTER_6,
STRIDE_6, PAD_6)
pb_pred = layer_xyf.convo_noneRelu(combine_LH, "conv_pb", FILTER_6,
STRIDE_6, PAD_6)
pc_pred = layer_xyf.convo_noneRelu(combine_LH, "conv_pc", FILTER_6,
STRIDE_6, PAD_6)
pd_pred = layer_xyf.convo_noneRelu(combine_LH, "conv_pd", FILTER_6,
STRIDE_6, PAD_6)
d1 = pa_pred * rh + pb_pred * rl
d2 = pc_pred * rh + pd_pred * rl
d1 = tf.clip_by_value(d1, 0, 5)
d2 = tf.clip_by_value(d2, 0, 5)
tf.summary.image('d1_pred', d1, 1)
tf.summary.image('d2_pred', d2, 1)
return d1, d2, rl, rh