def rnn(images, mask_true, num_layers, num_hidden, filter_size, stride=1,
seq_length=20, input_length=10, tln=True):
gen_images = []
lstm = []
cell = []
hidden = []
shape = images.get_shape().as_list()
output_channels = shape[-1]
for i in xrange(num_layers):
if i == 0:
num_hidden_in = num_hidden[num_layers-1]
else:
num_hidden_in = num_hidden[i-1]
new_cell = cslstm('lstm_'+str(i+1),
filter_size,
num_hidden_in,
num_hidden[i],
shape,
tln=tln)
lstm.append(new_cell)
cell.append(None)
hidden.append(None)
gradient_highway = ghu('highway', filter_size, num_hidden[0], tln=tln)
mem = None
z_t = None
for t in xrange(seq_length-1):
reuse = bool(gen_images)
with tf.variable_scope('predrnn_pp', reuse=reuse):
if t < input_length:
inputs = images[:,t]
else:
inputs = mask_true[:,t-10]*images[:,t] + (1-mask_true[:,t-10])*x_gen
hidden[0], cell[0], mem = lstm[0](inputs, hidden[0], cell[0], mem)
z_t = gradient_highway(hidden[0], z_t)
hidden[1], cell[1], mem = lstm[1](z_t, hidden[1], cell[1], mem)
for i in xrange(2, num_layers):
hidden[i], cell[i], mem = lstm[i](hidden[i-1], hidden[i], cell[i], mem)
x_gen = tf.layers.conv2d(inputs=hidden[num_layers-1],
filters=output_channels,
kernel_size=1,
strides=1,
padding='same',
name="back_to_pixel")
gen_images.append(x_gen)
gen_images = tf.stack(gen_images)
# [batch_size, seq_length, height, width, channels]
gen_images = tf.transpose(gen_images, [1,0,2,3,4])
loss = tf.nn.l2_loss(gen_images - images[:,1:])
#loss += tf.reduce_sum(tf.abs(gen_images - images[:,1:]))
return [gen_images, loss]
def __init__(self, batch, img_width, img_height, channels, filters, kernel_size, num_layers,
seqlength, inputlength, patch_size):
super(predrnn, self).__init__()
self.num_layers = num_layers
self.seqlength = seqlength
self.inputlength = inputlength
self.filters = filters
self.batch = batch
self.img_width = img_width
self.img_height = img_height
self.channels = channels
self.patch_size = patch_size
self.kernel_size = kernel_size
self.layer1 = cslstm(batch, img_width, img_height, channels, filters[0], kernel_size)
self.layer2 = ghu(filters[0], kernel_size)
self.layer3 = cslstm(batch, img_width, img_height, channels, filters[1], kernel_size)
self.layer4 = cslstm(batch, img_width, img_height, channels, filters[2], kernel_size)
self.layer5 = cslstm(batch, img_width, img_height, channels, filters[3], kernel_size)
# modify here
self.layer6 = layers.Conv2D(patch_size*patch_size, kernel_size=kernel_size, padding='same')
def rnn(images,
images_bw,
mask_true,
num_layers,
num_hidden,
filter_size,
stride=1,
seq_length=11,
input_length=5,
tln=True):
gen_images = []
lstm_fw = []
lstm_bw = []
cell_fw = []
cell_bw = []
hidden_fw = []
hidden_bw = []
shape = images.get_shape().as_list()
output_channels = shape[-1]
# Time Machine (put memory and hidden per layer)
tm_hidden_fw = [[None for i in range(seq_length)] for k in range(4)]
tm_hidden_bw = [[None for i in range(seq_length)] for k in range(4)]
tm_mem_fw = [[None for i in range(seq_length)] for k in range(4)]
tm_mem_bw = [[None for i in range(seq_length)] for k in range(4)]
## Create causal lstm unit
# Create forward causal lstm unit
for i in range(num_layers):
if i == 0:
num_hidden_in = num_hidden[num_layers - 1]
else:
num_hidden_in = num_hidden[i - 1]
new_cell = cslstm('lstm_fw_' + str(i + 1),
filter_size,
num_hidden_in,
num_hidden[i],
shape,
tln=tln)
lstm_fw.append(new_cell)
cell_fw.append(None)
hidden_fw.append(None)
# Create backward causal lstm unit
for i in range(num_layers):
if i == 0:
num_hidden_in = num_hidden[num_layers - 1]
else:
num_hidden_in = num_hidden[i - 1]
new_cell = cslstm('lstm_bw_' + str(i + 1),
filter_size,
num_hidden_in,
num_hidden[i],
shape,
tln=tln)
lstm_bw.append(new_cell)
cell_bw.append(None)
hidden_bw.append(None)
## Create GHU unit
# Create forward and backward GHU unit
gradient_highway_fw = ghu('highway_fw',
filter_size,
num_hidden[0],
tln=tln)
gradient_highway_bw = ghu('highway_bw',
filter_size,
num_hidden[0],
tln=tln)
## Create lstm memory output and GHU output
# Create forward memory output and GHU output
mem_fw = None
z_t_fw = None
# Create backward memory output and GHU output
mem_bw = None
z_t_bw = None
print("seq_length:{}".format(seq_length))
# Layer 1
for t in range(seq_length):
print("Layer 1")
print("t:{}".format(t))
# reuse = bool(gen_images)
# Layer 1 Forward
with tf.variable_scope('bi_cslstm_l1', reuse=tf.AUTO_REUSE):
# accroding mask_true replace with random noise
inputs_fw = mask_true[:, t] * images[:, t] + (
1 - mask_true[:, t]) * sample_Z((1 - mask_true[:, t]))
tf.summary.image('masktrue_fw',
reshape_patch_back_gen(mask_true[:, t], 4), 11)
tf.summary.image('input_fw', reshape_patch_back_gen(inputs_fw, 4),
11)
hidden_fw[0], cell_fw[0], mem_fw = lstm_fw[0](inputs_fw,
hidden_fw[0],
cell_fw[0], mem_fw)
z_t_fw = gradient_highway_fw(hidden_fw[0], z_t_fw)
tm_hidden_fw[0][t] = z_t_fw
tm_mem_fw[0][t] = mem_fw
# Layer 1 Backward
with tf.variable_scope('bi_cslstm_l1', reuse=tf.AUTO_REUSE):
# accroding mask_true replace with random noise
inputs_bw = mask_true[:, seq_length - t - 1] * images_bw[:, t] + (
1 - mask_true[:, seq_length - t - 1]) * sample_Z(
(1 - mask_true[:, seq_length - t - 1]))
hidden_bw[0], cell_bw[0], mem_bw = lstm_bw[0](inputs_bw,
hidden_bw[0],
cell_bw[0], mem_bw)
z_t_bw = gradient_highway_bw(hidden_bw[0], z_t_bw)
tm_hidden_bw[0][t] = z_t_bw
tm_mem_bw[0][t] = mem_bw
# Layer 2 only have 5 lstm
hiddenConcatConv_l2 = [None for i in range(seq_length // 2)]
memConcatConv_l2 = [None for i in range(seq_length // 2)]
for t in range(seq_length // 2):
print("Layer 2")
print("t:{}".format(t))
# Merge forward and backward output from layer 1
with tf.variable_scope('merge_l2', reuse=tf.AUTO_REUSE):
if t < (seq_length // 2 // 2):
hiddenConcat = tf.concat([
tm_hidden_fw[0][t * 2],
tm_hidden_bw[0][(seq_length // 2 - t - 1) * 2]
],
axis=-1)
hiddenConcatConv_l2[t] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[0][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='F_h_merge_l2')
memConcat = tf.concat([
tm_mem_fw[0][t * 2],
tm_mem_bw[0][(seq_length // 2 - t - 1) * 2]
],
axis=-1)
memConcatConv_l2[t] = tf.layers.conv2d(
inputs=memConcat,
filters=tm_mem_fw[0][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='F_m_merge_l2')
else:
hiddenConcat = tf.concat([
tm_hidden_bw[0][(seq_length // 2 - t - 1) * 2],
tm_hidden_fw[0][t * 2]
],
axis=-1)
hiddenConcatConv_l2[t] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[0][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='B_h_merge_l2')
memConcat = tf.concat([
tm_mem_bw[0][(seq_length // 2 - t - 1) * 2],
tm_mem_fw[0][t * 2]
],
axis=-1)
memConcatConv_l2[t] = tf.layers.conv2d(
inputs=memConcat,
filters=tm_mem_fw[0][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='B_m_merge_l2')
for t in range(seq_length // 2):
# Layer 2 Forward
with tf.variable_scope('bi_cslstm_l2', reuse=tf.AUTO_REUSE):
hidden_fw[1], cell_fw[1], mem_fw = lstm_fw[1](
hiddenConcatConv_l2[t], hidden_fw[1], cell_fw[1],
memConcatConv_l2[t])
tm_hidden_fw[1][t] = hidden_fw[1]
tm_mem_fw[1][t] = mem_fw
# Layer 2 Backward
with tf.variable_scope('bi_cslstm_l2', reuse=tf.AUTO_REUSE):
hidden_bw[1], cell_bw[1], mem_bw = lstm_bw[1](
hiddenConcatConv_l2[seq_length // 2 - t - 1], hidden_bw[1],
cell_bw[1], memConcatConv_l2[seq_length // 2 - t - 1])
tm_hidden_bw[1][t] = hidden_bw[1]
tm_mem_bw[1][t] = mem_bw
# Layer 3 only have 5 lstm
hiddenConcatConv_l3 = [None for i in range(seq_length // 2)]
memConcatConv_l3 = [None for i in range(seq_length // 2)]
for t in range(seq_length // 2):
print("Layer 3")
print("t:{}".format(t))
# Merge forward and backward output from layer 2
with tf.variable_scope('merge_l3', reuse=tf.AUTO_REUSE):
if t < (seq_length // 2 // 2):
hiddenConcat = tf.concat([
tm_hidden_fw[1][t],
tm_hidden_bw[1][seq_length // 2 - t - 1]
],
axis=-1)
hiddenConcatConv_l3[t] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[1][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='F_h_merge_l3')
memConcat = tf.concat(
[tm_mem_fw[1][t], tm_mem_bw[1][seq_length // 2 - t - 1]],
axis=-1)
memConcatConv_l3[t] = tf.layers.conv2d(
inputs=memConcat,
filters=tm_mem_fw[1][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='F_m_merge_l3')
else:
hiddenConcat = tf.concat([
tm_hidden_bw[1][seq_length // 2 - t - 1],
tm_hidden_fw[1][t]
],
axis=-1)
hiddenConcatConv_l3[t] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[1][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='B_h_merge_l3')
memConcat = tf.concat(
[tm_mem_bw[1][seq_length // 2 - t - 1], tm_mem_fw[1][t]],
axis=-1)
memConcatConv_l3[t] = tf.layers.conv2d(
inputs=memConcat,
filters=tm_mem_fw[1][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='B_m_merge_l3')
for t in range(seq_length // 2):
# Layer 3 Forward
with tf.variable_scope('bi_cslstm_l3', reuse=tf.AUTO_REUSE):
hidden_fw[2], cell_fw[2], mem_fw = lstm_fw[2](
hiddenConcatConv_l3[t], hidden_fw[2], cell_fw[2],
memConcatConv_l3[t])
tm_hidden_fw[2][t] = hidden_fw[2]
tm_mem_fw[2][t] = mem_fw
# Layer 3 Backward
with tf.variable_scope('bi_cslstm_l3', reuse=tf.AUTO_REUSE):
hidden_bw[2], cell_bw[2], mem_bw = lstm_bw[2](
hiddenConcatConv_l3[seq_length // 2 - t - 1], hidden_bw[2],
cell_bw[2], memConcatConv_l3[seq_length // 2 - t - 1])
tm_hidden_bw[2][t] = hidden_bw[2]
tm_mem_bw[2][t] = mem_bw
# Layer 4 only have 5 lstm
hiddenConcatConv_l4 = [None for i in range(seq_length // 2)]
memConcatConv_l4 = [None for i in range(seq_length // 2)]
for t in range(seq_length // 2):
print("Layer 4")
print("t:{}".format(t))
# Merge forward and backward output from layer 3
with tf.variable_scope('merge_l4', reuse=tf.AUTO_REUSE):
if t < (seq_length // 2 // 2):
hiddenConcat = tf.concat([
tm_hidden_fw[2][t],
tm_hidden_bw[2][seq_length // 2 - t - 1]
],
axis=-1)
hiddenConcatConv_l4[t] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[2][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='F_h_merge_l4')
memConcat = tf.concat(
[tm_mem_fw[2][t], tm_mem_bw[2][seq_length // 2 - t - 1]],
axis=-1)
memConcatConv_l4[t] = tf.layers.conv2d(
inputs=memConcat,
filters=tm_mem_fw[2][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='F_m_merge_l4')
else:
hiddenConcat = tf.concat([
tm_hidden_bw[2][seq_length // 2 - t - 1],
tm_hidden_fw[2][t]
],
axis=-1)
hiddenConcatConv_l4[t] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[2][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='B_h_merge_l4')
memConcat = tf.concat(
[tm_mem_bw[2][seq_length // 2 - t - 1], tm_mem_fw[2][t]],
axis=-1)
memConcatConv_l4[t] = tf.layers.conv2d(
inputs=memConcat,
filters=tm_mem_fw[2][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='B_m_merge_l4')
for t in range(seq_length // 2):
# Layer 4 Forward
with tf.variable_scope('bi_cslstm_l4', reuse=tf.AUTO_REUSE):
hidden_fw[3], cell_fw[3], mem_fw = lstm_fw[3](
hiddenConcatConv_l4[t], hidden_fw[3], cell_fw[3],
memConcatConv_l4[t])
tm_hidden_fw[3][t] = hidden_fw[3]
tm_mem_fw[3][t] = mem_fw
# Layer 4 Backward
with tf.variable_scope('bi_cslstm_l4', reuse=tf.AUTO_REUSE):
hidden_bw[3], cell_bw[3], mem_bw = lstm_bw[3](
hiddenConcatConv_l4[seq_length // 2 - t - 1], hidden_bw[3],
cell_bw[3], memConcatConv_l4[seq_length // 2 - t - 1])
tm_hidden_bw[3][t] = hidden_bw[3]
tm_mem_bw[3][t] = mem_bw
# generate output image
hiddenConcatConv = [None for i in range(seq_length // 2)]
x_gen = [None for i in range(seq_length // 2)]
for t in range(seq_length // 2):
with tf.variable_scope('bi_merge', reuse=tf.AUTO_REUSE):
if t < (seq_length // 2 // 2):
hiddenConcat = tf.concat([
tm_hidden_fw[3][t],
tm_hidden_bw[3][seq_length // 2 - t - 1]
],
axis=-1)
hiddenConcatConv[t] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[3][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='F_h_merge')
else:
hiddenConcat = tf.concat([
tm_hidden_bw[3][seq_length // 2 - t - 1],
tm_hidden_fw[3][t]
],
axis=-1)
hiddenConcatConv[t] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_bw[3][t].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name='B_h_merge')
for t in range(seq_length // 2):
with tf.variable_scope('generate', reuse=tf.AUTO_REUSE):
x_gen[t] = tf.layers.conv2d(inputs=hiddenConcatConv[t],
filters=output_channels,
kernel_size=1,
strides=1,
padding='same',
name='bi_back_to_pixel')
gen_images.append(x_gen[t])
print("generate t: %d" % t)
tf.summary.image('x_gen', reshape_patch_back_gen(x_gen[t], 4),
seq_length)
gen_images = tf.stack(gen_images)
# [batch_size, seq_length, height, width, channels]
gen_images = tf.transpose(gen_images, [1, 0, 2, 3, 4])
gt_images = [images[:, i * 2 + 1] for i in range(seq_length // 2)]
gt_images = tf.stack(gt_images)
gt_images = tf.transpose(gt_images, [1, 0, 2, 3, 4])
l2Loss = tf.nn.l2_loss(gen_images - gt_images)
l1Loss = tf.losses.absolute_difference(gen_images, gt_images)
gdlLoss = cal_gdl(gen_images, gt_images)
loss = l2Loss + l1Loss + gdlLoss
tf.summary.scalar('l2_Loss', l2Loss)
tf.summary.scalar('l1_Loss', l1Loss)
tf.summary.scalar('gdl_Loss', gdlLoss)
tf.summary.scalar('loss', loss)
return [gen_images, loss, images, images_bw]
def rnn(images,
mask_true,
num_layers,
num_hidden,
filter_size,
stride=1,
seq_length=20,
input_length=10,
tln=True,
batch_size=None):
gen_images = []
lstm = []
cell = []
hidden = []
shape = images.get_shape().as_list()
output_channels = shape[-1]
for i in range(num_layers):
if i == 0:
num_hidden_in = num_hidden[num_layers - 1]
else:
num_hidden_in = num_hidden[i - 1]
new_cell = cslstm('lstm_' + str(i + 1),
filter_size,
num_hidden_in,
num_hidden[i],
shape,
tln=tln,
batch_size=batch_size)
lstm.append(new_cell)
cell.append(None)
hidden.append(None)
gradient_highway = ghu('highway', filter_size, num_hidden[0], tln=tln)
mem = None
z_t = None
for t in range(seq_length - 1):
reuse = bool(gen_images)
with tf.variable_scope('predrnn_pp', reuse=reuse):
if t < input_length:
inputs = images[:, t, ...]
else:
inputs = mask_true[:, t - input_length,
...] * images[:, t, ...] + (
1 - mask_true[:, t - input_length,
...]) * x_gen
hidden[0], cell[0], mem = lstm[0](inputs, hidden[0], cell[0], mem)
z_t = gradient_highway(hidden[0], z_t, batch_size)
hidden[1], cell[1], mem = lstm[1](z_t, hidden[1], cell[1], mem)
# The output hidden here is the results of tanh * tahnh, which falls into the range of [-1, 1]
for i in range(2, num_layers):
hidden[i], cell[i], mem = lstm[i](hidden[i - 1], hidden[i],
cell[i], mem)
x_gen = tf.layers.conv2d(
inputs=hidden[num_layers - 1],
filters=output_channels,
kernel_size=1,
strides=1,
# activation=tf.nn.tanh,
padding='same',
name="back_to_pixel")
# squash
x_gen = tf.reshape(x_gen, [
-1, FLAGS.img_height, FLAGS.img_width,
FLAGS.patch_size_height * FLAGS.patch_size_width,
FLAGS.img_channel
])
x_gen = squash(x_gen, dim=-1) # makes a unit vector
x_gen = tf.reshape(x_gen, [
-1, FLAGS.img_height, FLAGS.img_width,
FLAGS.patch_size_height * FLAGS.patch_size_width *
FLAGS.img_channel
])
gen_images.append(x_gen)
gen_images = tf.stack(gen_images, axis=1)
gt_images = images[:, 1:]
gt_images = tf.reshape(gt_images, [
-1, FLAGS.seq_length - 1, FLAGS.img_height, FLAGS.img_width,
FLAGS.patch_size_height * FLAGS.patch_size_width, FLAGS.img_channel
])
gen_images = tf.reshape(gen_images, [
-1, FLAGS.seq_length - 1, FLAGS.img_height, FLAGS.img_width,
FLAGS.patch_size_height * FLAGS.patch_size_width, FLAGS.img_channel
])
# loss on the magnitude of speed
gt_speed = tf.sqrt(gt_images[..., 0]**2 + gt_images[..., 1]**2)
gen_speed = tf.sqrt(gen_images[..., 0]**2 + gen_images[..., 1]**2)
if FLAGS.loss_nan == 'nan':
loss = masked_mse_tf(gen_speed, gt_speed, null_val=np.nan)
loss += masked_mse_tf(gen_images, gt_images, null_val=np.nan)
else:
loss = masked_mse_tf(gen_speed, gt_speed, null_val=0.0)
loss += masked_mse_tf(gen_images, gt_images, null_val=0.0)
return [gen_images, loss]
def rnn_inference(images,
num_layers,
num_hidden,
filter_size,
stride=1,
pred_length=11,
input_length=1,
tln=True):
lstm = []
cell = []
hidden = []
shape = images.get_shape().as_list()
output_channels = shape[-1]
# input_length = tf.shape(images)[1]
for i in range(num_layers):
if i == 0:
num_hidden_in = num_hidden[num_layers - 1]
else:
num_hidden_in = num_hidden[i - 1]
new_cell = cslstm('lstm_' + str(i + 1),
filter_size,
num_hidden_in,
num_hidden[i],
shape,
tln=tln)
lstm.append(new_cell)
with tf.variable_scope('states_layer%d' % i, reuse=False) as scope:
try:
c = tf.get_variable('c', trainable=False)
cell.append(c)
except ValueError:
cell.append(None)
try:
h = tf.get_variable('h', trainable=False)
hidden.append(h)
except ValueError:
hidden.append(None)
gradient_highway = ghu('highway', filter_size, num_hidden[0], tln=tln)
with tf.variable_scope('states_global', reuse=False) as scope:
try:
mem = [tf.get_variable('mem', trainable=False)]
except ValueError:
mem = [None]
try:
z_t = [tf.get_variable('z_t', trainable=False)]
except ValueError:
z_t = [None]
x_gen = [None]
def step_forward(inputs):
with tf.variable_scope('predrnn_pp', reuse=tf.AUTO_REUSE):
hidden[0], cell[0], mem[0] = lstm[0](inputs, hidden[0], cell[0],
mem[0])
z_t[0] = gradient_highway(hidden[0], z_t[0])
hidden[1], cell[1], mem[0] = lstm[1](z_t[0], hidden[1], cell[1],
mem[0])
for i in range(2, num_layers):
hidden[i], cell[i], mem[0] = lstm[i](hidden[i - 1], hidden[i],
cell[i], mem[0])
x_gen[0] = tf.layers.conv2d(inputs=hidden[num_layers - 1],
filters=output_channels,
kernel_size=1,
strides=1,
padding='same',
name="back_to_pixel")
'''
t = tf.constant(0)
cond = lambda t: tf.less(t, input_length)
def body(t):
step_forward(images[:, t])
t += 1
tf.while_loop(cond, body, [t])
'''
for t in range(input_length):
step_forward(images[:, t])
for i in range(num_layers):
with tf.variable_scope('states_layer%d' % i,
reuse=tf.AUTO_REUSE) as scope:
h = tf.get_variable('h', hidden[i].get_shape(), trainable=False)
h.assign(hidden[i])
c = tf.get_variable('c', cell[i].get_shape(), trainable=False)
c.assign(cell[i])
with tf.variable_scope('states_global', reuse=tf.AUTO_REUSE) as scope:
m = tf.get_variable('mem', mem[0].get_shape(), trainable=False)
m.assign(mem[0])
z = tf.get_variable('z_t', z_t[0].get_shape(), trainable=False)
z.assign(z_t[0])
for i in range(pred_length - 1):
step_forward(x_gen[0])
return x_gen[0]
def rnn(images,
images_bw,
mask_true,
num_layers,
num_hidden,
filter_size,
stride=1,
seq_length=20,
input_length=5,
tln=True):
###'num_hidden', '32,16,16,16', 4 layers
# inp_images = []
gen_images = []
lstm_fw = []
lstm_bw = []
lstm_bi = []
cell_fw = []
cell_bw = []
cell_bi = []
hidden_fw = []
hidden_bw = []
hidden_bi = []
# shapeConcat = tf.concat([images, mask_true],axis=-1)
# shape = shapeConcat.get_shape().as_list()
# output_channels = shape[-1]/2
shape = images.get_shape().as_list()
output_channels = shape[-1]
# Time Machine
tm_hidden_fw = [[None for i in range(seq_length - 2)] for k in range(4)]
tm_hidden_bw = [[None for i in range(seq_length - 2)] for k in range(4)]
tm_mem_fw = [[None for i in range(seq_length - 2)] for k in range(4)]
tm_mem_bw = [[None for i in range(seq_length - 2)] for k in range(4)]
# loss_gdl = GDL()
for i in xrange(num_layers):
if i == 0:
num_hidden_in = num_hidden[
num_layers - 1] ### [4-1]=3 equels [i-1]=-1 equels 64
else:
num_hidden_in = num_hidden[i - 1]
new_cell_fw = bjstlstm('lstm_fw_' + str(i + 1),
filter_size,
num_hidden_in,
num_hidden[i],
shape,
tln=tln)
lstm_fw.append(new_cell_fw)
cell_fw.append(None)
hidden_fw.append(None)
for i in xrange(num_layers):
if i == 0:
num_hidden_in = num_hidden[
num_layers - 1] ### [4-1]=3 equels [i-1]=-1 equels 64
else:
num_hidden_in = num_hidden[i - 1]
new_cell_bw = bjstlstm('lstm_bw_' + str(i + 1),
filter_size,
num_hidden_in,
num_hidden[i],
shape,
tln=tln)
lstm_bw.append(new_cell_bw)
cell_bw.append(None)
hidden_bw.append(None)
# for i in xrange(num_layers-2, num_layers-1):
# num_hidden_in = num_hidden[i-1]
# new_bicell = bjstlstm('blstm_'+str(i+1),
# filter_size,
# num_hidden_in,
# num_hidden[i],
# shape,
# tln=tln)
# lstm_bi.append(new_bicell)
# cell_bi.append(None)
# hidden_bi.append(None)
gradient_highway_fw = ghu('highway_fw',
filter_size,
num_hidden[0],
tln=tln)
gradient_highway_bw = ghu('highway_bw',
filter_size,
num_hidden[0],
tln=tln)
mem_fw = None
z_t_fw = None
mem_bw = None
z_t_bw = None
for t_layer1 in xrange(seq_length - 2): ### t_layer1 = time
with tf.variable_scope('b_jstlstm_l1', reuse=tf.AUTO_REUSE):
inputs_fw = mask_true[:, t_layer1] * images[:, t_layer1] + (
1 - mask_true[:, t_layer1]) * sample_Z(
(1 - mask_true[:, t_layer1]))
# inputs_fwConcat = tf.concat([inputs_fw, mask_true[:,t_layer1]],axis=-1)
tf.summary.image('masktrue_fw',
reshape_patch_back_gen(mask_true[:, t_layer1], 4),
29)
tf.summary.image('input_fw', reshape_patch_back_gen(inputs_fw, 4),
29)
hidden_fw[0], cell_fw[0], mem_fw = lstm_fw[0](inputs_fw,
hidden_fw[0],
cell_fw[0], mem_fw)
z_t_fw = gradient_highway_fw(hidden_fw[0], z_t_fw)
tm_hidden_fw[0][t_layer1] = z_t_fw
tm_mem_fw[0][t_layer1] = mem_fw
with tf.variable_scope('b_jstlstm_l1', reuse=tf.AUTO_REUSE):
inputs_bw = mask_true[:, seq_length - 1 -
t_layer1] * images_bw[:, t_layer1] + (
1 -
mask_true[:, seq_length - 1 - t_layer1]
) * sample_Z((1 - mask_true[:, seq_length -
1 - t_layer1]))
hidden_bw[0], cell_bw[0], mem_bw = lstm_bw[0](inputs_bw,
hidden_bw[0],
cell_bw[0], mem_bw)
z_t_bw = gradient_highway_bw(hidden_bw[0], z_t_bw)
tm_hidden_bw[0][t_layer1] = z_t_bw
tm_mem_bw[0][t_layer1] = mem_bw
hiddenConcatConv_l2 = [None for i in range(seq_length - 4)]
memConcatConv_l2 = [None for i in range(seq_length - 4)]
for t_layer2 in xrange(seq_length - 4): ### t_layer2 = time
with tf.variable_scope('merge_l2', reuse=tf.AUTO_REUSE):
if t_layer2 < int((seq_length - 4) / 2):
hiddenConcat_bw = tf.concat([
tm_hidden_bw[0][t_layer2], tm_hidden_fw[0][-1 - t_layer2]
],
axis=-1)
hiddenConcatConv_l2[-1 - t_layer2] = tf.layers.conv2d(
inputs=hiddenConcat_bw,
filters=tm_hidden_fw[0][t_layer2].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="B_h_merge_l2")
memConcat_bw = tf.concat(
[tm_mem_bw[0][t_layer2], tm_mem_fw[0][-1 - t_layer2]],
axis=-1)
memConcatConv_l2[-1 - t_layer2] = tf.layers.conv2d(
inputs=memConcat_bw,
filters=tm_mem_fw[0][t_layer2].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="B_m_merge_l2")
hiddenConcat_fw = tf.concat([
tm_hidden_fw[0][t_layer2], tm_hidden_bw[0][-1 - t_layer2]
],
axis=-1)
hiddenConcatConv_l2[t_layer2] = tf.layers.conv2d(
inputs=hiddenConcat_fw,
filters=tm_hidden_fw[0][t_layer2].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="F_h_merge_l2")
memConcat_fw = tf.concat(
[tm_mem_fw[0][t_layer2], tm_mem_bw[0][-1 - t_layer2]],
axis=-1)
memConcatConv_l2[t_layer2] = tf.layers.conv2d(
inputs=memConcat_fw,
filters=tm_mem_fw[0][t_layer2].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="F_m_merge_l2")
with tf.variable_scope('b_jstlstm_l2', reuse=tf.AUTO_REUSE):
hidden_bw[1], cell_bw[1], mem_bw = lstm_bw[1](
hiddenConcatConv_l2[-1 - t_layer2], hidden_bw[1], cell_bw[1],
memConcatConv_l2[-1 - t_layer2])
tm_hidden_bw[1][t_layer2 + 1] = hidden_bw[1]
tm_mem_bw[1][t_layer2 + 1] = mem_bw
with tf.variable_scope('b_jstlstm_l2', reuse=tf.AUTO_REUSE):
hidden_fw[1], cell_fw[1], mem_fw = lstm_fw[1](
hiddenConcatConv_l2[t_layer2], hidden_fw[1], cell_fw[1],
memConcatConv_l2[t_layer2])
tm_hidden_fw[1][t_layer2 + 1] = hidden_fw[1]
tm_mem_fw[1][t_layer2 + 1] = mem_fw
hiddenConcatConv_l3 = [None for i in range(seq_length - 6)]
memConcatConv_l3 = [None for i in range(seq_length - 6)]
for t_layer3 in xrange(seq_length - 6): ### t_layer3 = time
with tf.variable_scope('merge_l3', reuse=tf.AUTO_REUSE):
if t_layer3 < int((seq_length - 6) / 2):
hiddenConcat_fw = tf.concat([
tm_hidden_fw[1][(t_layer3 + 1)],
tm_hidden_bw[1][-1 - (t_layer3 + 1)]
],
axis=-1)
hiddenConcatConv_l3[t_layer3] = tf.layers.conv2d(
inputs=hiddenConcat_fw,
filters=tm_hidden_fw[1][(t_layer3 + 1)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="F_h_merge_l3")
memConcat_fw = tf.concat([
tm_mem_fw[1][(t_layer3 + 1)], tm_mem_bw[1][-1 -
(t_layer3 + 1)]
],
axis=-1)
memConcatConv_l3[t_layer3] = tf.layers.conv2d(
inputs=memConcat_fw,
filters=tm_mem_fw[1][(t_layer3 + 1)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="F_m_merge_l3")
hiddenConcat_bw = tf.concat([
tm_hidden_bw[1][(t_layer3 + 1)],
tm_hidden_fw[1][-1 - (t_layer3 + 1)]
],
axis=-1)
hiddenConcatConv_l3[-1 - t_layer3] = tf.layers.conv2d(
inputs=hiddenConcat_bw,
filters=tm_hidden_fw[1][(t_layer3 + 1)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="B_h_merge_l3")
memConcat_bw = tf.concat([
tm_mem_bw[1][(t_layer3 + 1)], tm_mem_fw[1][-1 -
(t_layer3 + 1)]
],
axis=-1)
memConcatConv_l3[-1 - t_layer3] = tf.layers.conv2d(
inputs=memConcat_bw,
filters=tm_mem_fw[1][(t_layer3 + 1)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="B_m_merge_l3")
with tf.variable_scope('b_jstlstm_l3', reuse=tf.AUTO_REUSE):
hidden_fw[2], cell_fw[2], mem_fw = lstm_fw[2](
hiddenConcatConv_l3[t_layer3], hidden_fw[2], cell_fw[2],
memConcatConv_l3[t_layer3])
tm_hidden_fw[2][t_layer3 + 2] = hidden_fw[2]
tm_mem_fw[2][t_layer3 + 2] = mem_fw
with tf.variable_scope('b_jstlstm_l3', reuse=tf.AUTO_REUSE):
hidden_bw[2], cell_bw[2], mem_bw = lstm_bw[2](
hiddenConcatConv_l3[-1 - t_layer3], hidden_bw[2], cell_bw[2],
memConcatConv_l3[-1 - t_layer3])
tm_hidden_bw[2][t_layer3 + 2] = hidden_bw[2]
tm_mem_bw[2][t_layer3 + 2] = mem_bw
hiddenConcatConv_l4 = [None for i in range(seq_length - 8)]
memConcatConv_l4 = [None for i in range(seq_length - 8)]
hiddenConcatConv = [None for i in range(seq_length - 8)]
for t_layer4 in xrange(seq_length - 8): ### t_layer4 = time
with tf.variable_scope('merge_l4', reuse=tf.AUTO_REUSE):
if t_layer4 < int((seq_length - 8) / 2):
hiddenConcat_bw = tf.concat([
tm_hidden_bw[2][(t_layer4 + 2)],
tm_hidden_fw[2][-1 - (t_layer4 + 2)]
],
axis=-1)
hiddenConcatConv_l4[-1 - t_layer4] = tf.layers.conv2d(
inputs=hiddenConcat_bw,
filters=tm_hidden_fw[2][(t_layer4 + 2)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="B_h_merge_l4")
memConcat_bw = tf.concat([
tm_mem_bw[2][(t_layer4 + 2)], tm_mem_fw[2][-1 -
(t_layer4 + 2)]
],
axis=-1)
memConcatConv_l4[-1 - t_layer4] = tf.layers.conv2d(
inputs=memConcat_bw,
filters=tm_mem_fw[2][(t_layer4 + 2)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="B_m_merge_l4")
hiddenConcat_fw = tf.concat([
tm_hidden_fw[2][(t_layer4 + 2)],
tm_hidden_bw[2][-1 - (t_layer4 + 2)]
],
axis=-1)
hiddenConcatConv_l4[t_layer4] = tf.layers.conv2d(
inputs=hiddenConcat_fw,
filters=tm_hidden_fw[2][(t_layer4 + 2)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="F_h_merge_l4")
memConcat_fw = tf.concat([
tm_mem_fw[2][(t_layer4 + 2)], tm_mem_bw[2][-1 -
(t_layer4 + 2)]
],
axis=-1)
memConcatConv_l4[t_layer4] = tf.layers.conv2d(
inputs=memConcat_fw,
filters=tm_mem_fw[2][(t_layer4 + 2)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="F_m_merge_l4")
with tf.variable_scope('b_jstlstm_l4', reuse=tf.AUTO_REUSE):
hidden_bw[3], cell_bw[3], mem_bw = lstm_bw[3](
hiddenConcatConv_l4[-1 - t_layer4], hidden_bw[3], cell_bw[3],
memConcatConv_l4[-1 - t_layer4])
tm_hidden_bw[3][t_layer4 + 3] = hidden_bw[3]
tm_mem_bw[3][t_layer4 + 3] = mem_bw
with tf.variable_scope('b_jstlstm_l4', reuse=tf.AUTO_REUSE):
hidden_fw[3], cell_fw[3], mem_fw = lstm_fw[3](
hiddenConcatConv_l4[t_layer4], hidden_fw[3], cell_fw[3],
memConcatConv_l4[t_layer4])
tm_hidden_fw[3][t_layer4 + 3] = hidden_fw[3]
tm_mem_fw[3][t_layer4 + 3] = mem_fw
x_gen = [None for i in range(seq_length - 8)]
for t_bi in xrange(seq_length - 8): ### t_bi = time
with tf.variable_scope('bi_merge', reuse=tf.AUTO_REUSE):
if t_bi < int((seq_length - 8) / 2):
hiddenConcat = tf.concat([
tm_hidden_fw[3][(t_bi + 3)], tm_hidden_bw[3][-1 -
(t_bi + 3)]
],
axis=-1)
hiddenConcatConv[t_bi] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[3][(t_bi + 3)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="F_h_merge")
hiddenConcat = tf.concat([
tm_hidden_bw[3][(t_bi + 3)], tm_hidden_fw[3][-1 -
(t_bi + 3)]
],
axis=-1)
hiddenConcatConv[-1 - t_bi] = tf.layers.conv2d(
inputs=hiddenConcat,
filters=tm_hidden_fw[3][(t_bi + 3)].get_shape()[-1],
kernel_size=1,
strides=1,
padding='same',
name="B_h_merge")
x_gen[t_bi] = tf.layers.conv2d(inputs=hiddenConcatConv[t_bi],
filters=output_channels,
kernel_size=1,
strides=1,
padding='same',
name="bi_back_to_pixel")
x_gen[-1 - t_bi] = tf.layers.conv2d(
inputs=hiddenConcatConv[-1 - t_bi],
filters=output_channels,
kernel_size=1,
strides=1,
padding='same',
name="bi_back_to_pixel")
gen_images.append(x_gen[t_bi])
print("t_bi: %d" % t_bi)
tf.summary.image('x_gen', reshape_patch_back_gen(x_gen[t_bi], 4),
seq_length - 2)
# inp_images = tf.stack(inp_images)
gen_images = tf.stack(gen_images)
# # # gen_images_bw = tf.stack(gen_images_bw)
# [batch_size, seq_length, height, width, channels]
# inp_images = tf.transpose(inp_images, [1,0,2,3,4])
gen_images = tf.transpose(gen_images, [1, 0, 2, 3, 4])
# # # gen_images_bw = tf.transpose(gen_images_bw, [1,0,2,3,4])
# No 0, 29, from 1 to seq_length-2
l2Loss = tf.nn.l2_loss(gen_images - images[:, 4:-4])
l1Loss = tf.losses.absolute_difference(gen_images, images[:, 4:-4])
#hbloss = tf.losses.huber_loss(images[:,4:-4], gen_images, delta=1.5)
gdlLoss = cal_gdl(gen_images, images[:, 4:-4])
loss = l2Loss + l1Loss + gdlLoss
tf.summary.scalar('l2_Loss', l2Loss)
tf.summary.scalar('l1_Loss', l1Loss)
#tf.summary.scalar('huber_Loss', hbloss)
tf.summary.scalar('gdl_Loss', gdlLoss)
tf.summary.scalar('loss', loss)
return [gen_images, loss, images, images_bw]
