def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1.params)
pool1_ = PoolLayer(conv1_)
rect1_ = LeakyReLU(pool1_)
conv2_ = ConvLayer(rect1_, (n_convfilter[1], 3, 3),
params=conv2.params)
pool2_ = PoolLayer(conv2_)
rect2_ = LeakyReLU(pool2_)
conv3_ = ConvLayer(rect2_, (n_convfilter[2], 3, 3),
params=conv3.params)
pool3_ = PoolLayer(conv3_)
rect3_ = LeakyReLU(pool3_)
conv4_ = ConvLayer(rect3_, (n_convfilter[3], 3, 3),
params=conv4.params)
pool4_ = PoolLayer(conv4_)
rect4_ = LeakyReLU(pool4_)
conv5_ = ConvLayer(rect4_, (n_convfilter[4], 3, 3),
params=conv5.params)
pool5_ = PoolLayer(conv5_)
rect5_ = LeakyReLU(pool5_)
conv6_ = ConvLayer(rect5_, (n_convfilter[5], 3, 3),
params=conv6.params)
pool6_ = PoolLayer(conv6_)
rect6_ = LeakyReLU(pool6_)
flat6_ = FlattenLayer(rect6_)
fc7_ = TensorProductLayer(flat6_,
n_fc_filters[0],
params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_.output, update_gate_.output
def gru4(curr_s4, prev_s4):
curr_s4 = tensor.reshape(curr_s4, s_shape4)
curr_s4_ = InputLayer(s_shape4, curr_s4)
prev_s4_ = InputLayer(s_shape4, prev_s4)
t_x_s_update4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_update4.params)
t_x_s_reset4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_reset4.params)
update4_ = SigmoidLayer(t_x_s_update4_)
comp_udpate_gate4_ = ComplementLayer(update4_)
reset_gate4_ = SigmoidLayer(t_x_s_reset4_)
rs4_ = EltwiseMultiplyLayer(reset_gate4_, prev_s4_)
t_x_rs4_ = CConv3DLayer(rs4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_rs4.params)
tanh_t_x_rs4_ = TanhLayer(t_x_rs4_)
gru_out4_ = AddLayer(
EltwiseMultiplyLayer(update4_, prev_s4_),
EltwiseMultiplyLayer(comp_udpate_gate4_, tanh_t_x_rs4_))
print("gru_out4: ", gru_out4_.output_shape)
return gru_out4_.output
def gru5(curr_s5, prev_s5):
curr_s5 = tensor.reshape(curr_s5, s_shape5)
curr_s5_ = InputLayer(s_shape5, curr_s5)
prev_s5_ = InputLayer(s_shape5, prev_s5)
print("curr_s5: ", curr_s5_.output_shape)
print("prev_s5: ", prev_s5_.output_shape)
t_x_s_update5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_s_update5.params)
t_x_s_reset5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_s_reset5.params)
update5_ = SigmoidLayer(t_x_s_update5_)
comp_udpate_gate5_ = ComplementLayer(update5_)
reset_gate5_ = SigmoidLayer(t_x_s_reset5_)
rs5_ = EltwiseMultiplyLayer(reset_gate5_, prev_s5_)
t_x_rs5_ = CConv3DLayer(rs5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_rs5.params)
tanh_t_x_rs5_ = TanhLayer(t_x_rs5_)
print("t_x_s_update5: ", t_x_s_update5_.output_shape)
print("t_x_s_reset5: ", t_x_s_reset5_.output_shape)
gru_out5_ = AddLayer(
EltwiseMultiplyLayer(update5_, prev_s5_),
EltwiseMultiplyLayer(comp_udpate_gate5_, tanh_t_x_rs5_))
print("gru_out5: ", gru_out5_.output_shape)
return gru_out5_.output
def decode_recurrence_2(x_curr, prev_s_tensor, prev_in_gate_tensor):
x_curr_ = InputLayer(fc_shape, x_curr)
prev_s_2_ = InputLayer(s_shape_2, prev_s_tensor)
t_x_s_update_2_ = FCConv3DLayer(
prev_s_2_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_update_2.params)
t_x_s_reset_2_ = FCConv3DLayer(
prev_s_2_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_reset_2.params)
update_gate_ = SigmoidLayer(t_x_s_update_2_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_2_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_2_)
t_x_rs_2_ = FCConv3DLayer(
rs_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_rs_2.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_2_)
gru_out_2_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_2_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_2_.output, update_gate_.output
def gru3(curr_s3, prev_s3):
curr_s3 = tensor.reshape(curr_s3, s_shape3)
curr_s3_ = InputLayer(s_shape3, curr_s3)
prev_s3_ = InputLayer(s_shape3, prev_s3)
t_x_s_update3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_s_update3.params)
t_x_s_reset3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_s_reset3.params)
update3_ = SigmoidLayer(t_x_s_update3_)
comp_udpate_gate3_ = ComplementLayer(update3_)
reset_gate3_ = SigmoidLayer(t_x_s_reset3_)
rs3_ = EltwiseMultiplyLayer(reset_gate3_, prev_s3_)
t_x_rs3_ = CConv3DLayer(rs3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_rs3.params)
tanh_t_x_rs3_ = TanhLayer(t_x_rs3_)
gru_out3_ = AddLayer(
EltwiseMultiplyLayer(update3_, prev_s3_),
EltwiseMultiplyLayer(comp_udpate_gate3_, tanh_t_x_rs3_))
print("gru_out3: ", gru_out3_.output_shape)
return gru_out3_.output
def gru2(curr_s2, prev_s2):
curr_s2 = tensor.reshape(curr_s2, s_shape2)
curr_s2_ = InputLayer(s_shape2, curr_s2)
prev_s2_ = InputLayer(s_shape2, prev_s2)
t_x_s_update2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_s_update2.params)
t_x_s_reset2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_s_reset2.params)
update2_ = SigmoidLayer(t_x_s_update2_)
comp_udpate_gate2_ = ComplementLayer(update2_)
reset_gate2_ = SigmoidLayer(t_x_s_reset2_)
rs2_ = EltwiseMultiplyLayer(reset_gate2_, prev_s2_)
t_x_rs2_ = CConv3DLayer(rs2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_rs2.params)
tanh_t_x_rs2_ = TanhLayer(t_x_rs2_)
gru_out2_ = AddLayer(
EltwiseMultiplyLayer(update2_, prev_s2_),
EltwiseMultiplyLayer(comp_udpate_gate2_, tanh_t_x_rs2_))
print("gru_out2: ", gru_out2_.output_shape)
return gru_out2_.output
def __init__(self, input_size, output_size, hidden_layer_sizes):
self.learning_rate = 0.1
self.input_layer = InputLayer(input_size)
self.output_layer = OutputLayer(output_size)
self.hidden_layers = [
HiddenLayer(hidden_layer_size)
for hidden_layer_size in hidden_layer_sizes
]
for i, hidden_layer in enumerate(self.hidden_layers):
if i == 0 and i == len(self.hidden_layers) - 1:
hidden_layer.initialize(self.input_layer, self.output_layer)
elif i == 0:
hidden_layer.initialize(self.input_layer,
self.hidden_layers[i + 1])
elif i == len(self.hidden_layers) - 1:
hidden_layer.initialize(self.hidden_layers[i - 1],
self.output_layer)
else:
hidden_layer.initialize(self.hidden_layers[i - 1],
self.hidden_layers[i + 1])
if (len(self.hidden_layers)):
self.output_layer.initialize(self.hidden_layers[-1])
else:
self.output_layer.initialize(self.input_layer)
class FeedForwardNeuralNetwork:
def __init__(self, input_size, output_size, hidden_layer_sizes):
self.learning_rate = 0.1
self.input_layer = InputLayer(input_size)
self.output_layer = OutputLayer(output_size)
self.hidden_layers = [
HiddenLayer(hidden_layer_size)
for hidden_layer_size in hidden_layer_sizes
]
for i, hidden_layer in enumerate(self.hidden_layers):
if i == 0 and i == len(self.hidden_layers) - 1:
hidden_layer.initialize(self.input_layer, self.output_layer)
elif i == 0:
hidden_layer.initialize(self.input_layer,
self.hidden_layers[i + 1])
elif i == len(self.hidden_layers) - 1:
hidden_layer.initialize(self.hidden_layers[i - 1],
self.output_layer)
else:
hidden_layer.initialize(self.hidden_layers[i - 1],
self.hidden_layers[i + 1])
if (len(self.hidden_layers)):
self.output_layer.initialize(self.hidden_layers[-1])
else:
self.output_layer.initialize(self.input_layer)
def predict(self, input_arr):
self.input_layer.set_values(input_arr)
for hidden_layer in self.hidden_layers:
hidden_layer.feed_forward()
self.output_layer.feed_forward()
return self.output_layer.values
def train(self, input_arr, target_arr):
self.predict(input_arr)
self.output_layer.calculate_errors(target_arr)
for hidden_layer in reversed(self.hidden_layers):
hidden_layer.calculate_errors()
self.output_layer.adjust_parameters(self.learning_rate)
for hidden_layer in reversed(self.hidden_layers):
hidden_layer.adjust_parameters(self.learning_rate)
def encoder(x):
input_ = InputLayer(input_shape, x)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params)
rect1b_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1b_, padding=(0, 0))
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params)
rect2b_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2b_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params)
rect3b_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3b_)
pool3_ = PoolLayer(res3_, padding=(0, 0))
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params)
rect4b_ = LeakyReLU(conv4b_)
conv4c_ = ConvLayer(pool3_, (n_convfilter[3], 1, 1), params=conv4c.params)
res4_ = AddLayer(conv4c_, rect4b_)
pool4_ = PoolLayer(res4_, padding=(0, 0))
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params)
rect5b_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5b_)
pool5_ = PoolLayer(res5_, padding=(0, 0))
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params)
rect6b_ = LeakyReLU(conv6b_)
conv6c_ = ConvLayer(pool5_, (n_convfilter[5], 1, 1), params=conv6c.params)
res6_ = AddLayer(conv6c_, rect6b_)
flat3_ = FlattenLayer(res3_)
flat4_ = FlattenLayer(res4_)
flat5_ = FlattenLayer(res5_)
flat6_ = FlattenLayer(res6_)
print("res3: ", res3_.output_shape)
print("res4: ", res4_.output_shape)
print("res5: ", res5_.output_shape)
print("res6: ", res6_.output_shape)
# pool6_ = PoolLayer(res6_)
return flat3_.output, flat4_.output, flat5_.output, flat6_.output
def encode_recurrence(x_curr):
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3),
params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
# flat1_ = FlattenLayer(pool1_)
# fc1_ = TensorProductLayer(flat1_, n_fc_filters[0], params=fc1.params)
# out1_ = LeakyReLU(fc1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3),
params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3),
params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1),
params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
flat2_ = FlattenLayer(pool2_)
fc2_ = TensorProductLayer(flat2_,
n_fc_filters[0],
params=fc2.params)
out2_ = LeakyReLU(fc2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3),
params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3),
params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1),
params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
flat3_ = FlattenLayer(pool3_)
fc3_ = TensorProductLayer(flat3_,
n_fc_filters[0],
params=fc3.params)
out3_ = LeakyReLU(fc3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3),
params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3),
params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
flat4_ = FlattenLayer(pool4_)
fc4_ = TensorProductLayer(flat4_,
n_fc_filters[0],
params=fc4.params)
out4_ = LeakyReLU(fc4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3),
params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3),
params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1),
params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
flat5_ = FlattenLayer(pool5_)
fc5_ = TensorProductLayer(flat5_,
n_fc_filters[0],
params=fc5.params)
out5_ = LeakyReLU(fc5_)
return out5_.output, out4_.output, out3_.output, out2_.output # , out1_.output
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # n_deconvfilter = [128, 128, 128, 64, 32, 2]
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params)
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
flat6_ = FlattenLayer(pool6_)
fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) # (B, 128, 4, 4, 4)
return gru_out_.output, update_gate_.output
def network_definition(self):
self.x = tensor5()
self.is_x_tensor4 = False
img_w = self.img_w
img_h = self.img_h
n_gru_vox = [32, 32, 16, 8, 4]
n_convfilter = [16, 32, 64, 128, 256, 512]
n_deconvfilter = [2, 2, 8, 32, 128]
input_shape = (self.batch_size, 3, img_w, img_h)
x = InputLayer(input_shape)
conv1a = ConvLayer(x, (n_convfilter[0], 7, 7))
conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3))
pool1 = PoolLayer(conv1b, padding=(0, 0)) # H/2->64
conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3))
conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3))
conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1))
pool2 = PoolLayer(conv2c) # H/4->32
conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3))
conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3))
conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1))
pool3 = PoolLayer(conv3c, padding=(0, 0)) # H/8->16
conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3))
conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3))
conv4c = ConvLayer(pool3, (n_convfilter[3], 1, 1))
pool4 = PoolLayer(conv4c, padding=(0, 0)) # H/16->8
conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3))
conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3))
conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1)) # H/32->4
pool5 = PoolLayer(conv5c, padding=(0, 0))
conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3))
conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3))
conv6c = ConvLayer(pool5, (n_convfilter[5], 1, 1)) # H/32->4
def encoder(x):
input_ = InputLayer(input_shape, x)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params)
rect1b_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1b_, padding=(0, 0))
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params)
rect2b_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2b_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params)
rect3b_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3b_)
pool3_ = PoolLayer(res3_, padding=(0, 0))
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params)
rect4b_ = LeakyReLU(conv4b_)
conv4c_ = ConvLayer(pool3_, (n_convfilter[3], 1, 1), params=conv4c.params)
res4_ = AddLayer(conv4c_, rect4b_)
pool4_ = PoolLayer(res4_, padding=(0, 0))
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params)
rect5b_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5b_)
pool5_ = PoolLayer(res5_, padding=(0, 0))
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params)
rect6b_ = LeakyReLU(conv6b_)
conv6c_ = ConvLayer(pool5_, (n_convfilter[5], 1, 1), params=conv6c.params)
res6_ = AddLayer(conv6c_, rect6b_)
flat3_ = FlattenLayer(res3_)
flat4_ = FlattenLayer(res4_)
flat5_ = FlattenLayer(res5_)
flat6_ = FlattenLayer(res6_)
print("res3: ", res3_.output_shape)
print("res4: ", res4_.output_shape)
print("res5: ", res5_.output_shape)
print("res6: ", res6_.output_shape)
# pool6_ = PoolLayer(res6_)
return flat3_.output, flat4_.output, flat5_.output, flat6_.output
# return flat6_.output
# Set the shape of each resolution
s_shape5 = (self.batch_size, n_gru_vox[4], n_deconvfilter[4], n_gru_vox[4], n_gru_vox[4])
s_shape4 = (self.batch_size, n_gru_vox[3], n_deconvfilter[3], n_gru_vox[3], n_gru_vox[3])
s_shape3 = (self.batch_size, n_gru_vox[2], n_deconvfilter[2], n_gru_vox[2], n_gru_vox[2])
s_shape2 = (self.batch_size, n_gru_vox[1], n_deconvfilter[1], n_gru_vox[1], n_gru_vox[1])
## resolution 5
prev_s5 = InputLayer(s_shape5)
curr_s5 = InputLayer(s_shape5)
t_x_s_update5 = CConv3DLayer(prev_s5, curr_s5, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3))
t_x_s_reset5 = CConv3DLayer(prev_s5, curr_s5, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3))
reset_gate5 = SigmoidLayer(t_x_s_reset5)
rs5 = EltwiseMultiplyLayer(reset_gate5, prev_s5)
t_x_rs5 = CConv3DLayer(rs5, curr_s5, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3))
## resolution 4
prev_s4 = InputLayer(s_shape4)
curr_s4 = InputLayer(s_shape4)
t_x_s_update4 = CConv3DLayer(prev_s4, curr_s4, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3))
t_x_s_reset4 = CConv3DLayer(prev_s4, curr_s4, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3))
reset_gate4 = SigmoidLayer(t_x_s_reset4)
rs4 = EltwiseMultiplyLayer(reset_gate4, prev_s4)
t_x_rs4 = CConv3DLayer(rs4, curr_s4, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3))
# resolution 3
prev_s3 = InputLayer(s_shape3)
curr_s3 = InputLayer(s_shape3)
t_x_s_update3 = CConv3DLayer(prev_s3, curr_s3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3))
t_x_s_reset3 = CConv3DLayer(prev_s3, curr_s3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3))
reset_gate3 = SigmoidLayer(t_x_s_reset3)
rs3 = EltwiseMultiplyLayer(reset_gate3, prev_s3)
t_x_rs3 = CConv3DLayer(rs3, curr_s3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3))
# resolution 4
prev_s2 = InputLayer(s_shape2)
curr_s2 = InputLayer(s_shape2)
t_x_s_update2 = CConv3DLayer(prev_s2, curr_s2, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3))
t_x_s_reset2 = CConv3DLayer(prev_s2, curr_s2, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3))
reset_gate2 = SigmoidLayer(t_x_s_reset2)
rs2 = EltwiseMultiplyLayer(reset_gate2, prev_s2)
t_x_rs2 = CConv3DLayer(rs2, curr_s2, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3))
def gru5(curr_s5, prev_s5):
curr_s5 = tensor.reshape(curr_s5, s_shape5)
curr_s5_ = InputLayer(s_shape5, curr_s5)
prev_s5_ = InputLayer(s_shape5, prev_s5)
print("curr_s5: ", curr_s5_.output_shape)
print("prev_s5: ", prev_s5_.output_shape)
t_x_s_update5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_s_update5.params)
t_x_s_reset5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_s_reset5.params)
update5_ = SigmoidLayer(t_x_s_update5_)
comp_udpate_gate5_ = ComplementLayer(update5_)
reset_gate5_ = SigmoidLayer(t_x_s_reset5_)
rs5_ = EltwiseMultiplyLayer(reset_gate5_, prev_s5_)
t_x_rs5_ = CConv3DLayer(rs5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_rs5.params)
tanh_t_x_rs5_ = TanhLayer(t_x_rs5_)
print("t_x_s_update5: ", t_x_s_update5_.output_shape)
print("t_x_s_reset5: ", t_x_s_reset5_.output_shape)
gru_out5_ = AddLayer(
EltwiseMultiplyLayer(update5_, prev_s5_),
EltwiseMultiplyLayer(comp_udpate_gate5_, tanh_t_x_rs5_))
print("gru_out5: ", gru_out5_.output_shape)
return gru_out5_.output
def gru4(curr_s4, prev_s4):
curr_s4 = tensor.reshape(curr_s4, s_shape4)
curr_s4_ = InputLayer(s_shape4, curr_s4)
prev_s4_ = InputLayer(s_shape4, prev_s4)
t_x_s_update4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_update4.params)
t_x_s_reset4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_reset4.params)
update4_ = SigmoidLayer(t_x_s_update4_)
comp_udpate_gate4_ = ComplementLayer(update4_)
reset_gate4_ = SigmoidLayer(t_x_s_reset4_)
rs4_ = EltwiseMultiplyLayer(reset_gate4_, prev_s4_)
t_x_rs4_ = CConv3DLayer(rs4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_rs4.params)
tanh_t_x_rs4_ = TanhLayer(t_x_rs4_)
gru_out4_ = AddLayer(
EltwiseMultiplyLayer(update4_, prev_s4_),
EltwiseMultiplyLayer(comp_udpate_gate4_, tanh_t_x_rs4_))
print("gru_out4: ", gru_out4_.output_shape)
return gru_out4_.output
def gru3(curr_s3, prev_s3):
curr_s3 = tensor.reshape(curr_s3, s_shape3)
curr_s3_ = InputLayer(s_shape3, curr_s3)
prev_s3_ = InputLayer(s_shape3, prev_s3)
t_x_s_update3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_s_update3.params)
t_x_s_reset3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_s_reset3.params)
update3_ = SigmoidLayer(t_x_s_update3_)
comp_udpate_gate3_ = ComplementLayer(update3_)
reset_gate3_ = SigmoidLayer(t_x_s_reset3_)
rs3_ = EltwiseMultiplyLayer(reset_gate3_, prev_s3_)
t_x_rs3_ = CConv3DLayer(rs3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_rs3.params)
tanh_t_x_rs3_ = TanhLayer(t_x_rs3_)
gru_out3_ = AddLayer(
EltwiseMultiplyLayer(update3_, prev_s3_),
EltwiseMultiplyLayer(comp_udpate_gate3_, tanh_t_x_rs3_))
print("gru_out3: ", gru_out3_.output_shape)
return gru_out3_.output
def gru2(curr_s2, prev_s2):
curr_s2 = tensor.reshape(curr_s2, s_shape2)
curr_s2_ = InputLayer(s_shape2, curr_s2)
prev_s2_ = InputLayer(s_shape2, prev_s2)
t_x_s_update2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_s_update2.params)
t_x_s_reset2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_s_reset2.params)
update2_ = SigmoidLayer(t_x_s_update2_)
comp_udpate_gate2_ = ComplementLayer(update2_)
reset_gate2_ = SigmoidLayer(t_x_s_reset2_)
rs2_ = EltwiseMultiplyLayer(reset_gate2_, prev_s2_)
t_x_rs2_ = CConv3DLayer(rs2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_rs2.params)
tanh_t_x_rs2_ = TanhLayer(t_x_rs2_)
gru_out2_ = AddLayer(
EltwiseMultiplyLayer(update2_, prev_s2_),
EltwiseMultiplyLayer(comp_udpate_gate2_, tanh_t_x_rs2_))
print("gru_out2: ", gru_out2_.output_shape)
return gru_out2_.output
s_encoder, _ = theano.scan(encoder,
sequences=[self.x])
# print("self.x: ", self.x)
out_encoder5 = s_encoder[3]
out_encoder4 = s_encoder[2]
out_encoder3 = s_encoder[1]
out_encoder2 = s_encoder[0]
s_gru5, _ = theano.scan(gru5,
sequences=[out_encoder5],
outputs_info=[tensor.zeros_like(np.zeros(s_shape5),
dtype=theano.config.floatX)])
input_5 = InputLayer(s_shape5, s_gru5[-1])
# print("input_5: ", input_5.output_shape)
pred5 = Conv3DLayer(input_5, (2, 3, 3, 3))
unpool5 = Unpool3DLayer(input_5)
conv3d5 = Conv3DLayer(unpool5, (n_deconvfilter[3], 3, 3, 3))
rect3d5 = LeakyReLU(conv3d5)
print("rect3d5: ", rect3d5.output_shape)
print("recct3d5: ", rect3d5.output)
s_gru4, _ = theano.scan(gru4,
sequences=[out_encoder4],
outputs_info=[rect3d5.output]
)
input_4 = InputLayer(s_shape4, s_gru4[-1])
pred4 = Conv3DLayer(input_4, (2, 3, 3, 3))
unpool4 = Unpool3DLayer(input_4)
conv3d4 = Conv3DLayer(unpool4, (n_deconvfilter[2], 3, 3, 3))
rect3d4 = LeakyReLU(conv3d4)
print("rect3d4: ", rect3d4.output_shape)
print("recct3d4: ", rect3d4.output)
s_gru3, _ = theano.scan(gru3,
sequences=[out_encoder3],
outputs_info=[rect3d4.output])
input_3 = InputLayer(s_shape3, s_gru3[-1])
pred3 = Conv3DLayer(input_3, (2, 3, 3, 3))
unpool3 = Unpool3DLayer(input_3)
conv3d3 = Conv3DLayer(unpool3, (n_deconvfilter[1], 3, 3, 3))
rect3d3 = LeakyReLU(conv3d3)
print("rect3d3: ", rect3d3.output_shape)
print("recct3d3: ", rect3d3.output)
s_gru2, _ = theano.scan(gru2,
sequences=[out_encoder2],
outputs_info=[rect3d3.output])
input_2 = InputLayer(s_shape2, s_gru2[-1])
pred2 = Conv3DLayer(input_2, (2, 3, 3, 3))
labele_shape = self.y.shape
label3 = self.y[:, 0:labele_shape[1]:2, :, 0:labele_shape[3]:2, 0:labele_shape[4]:2]
label4 = self.y[:, 0:labele_shape[1]:4, :, 0:labele_shape[3]:4, 0:labele_shape[4]:4]
label5 = self.y[:, 0:labele_shape[1]:8, :, 0:labele_shape[3]:8, 0:labele_shape[4]:8]
# print("pred5: ", pred5.output_shape)
# print("pred4: ", pred4.output_shape)
# print("pred3: ", pred3.output_shape)
print("pred2: ", pred2.output_shape)
softmax_loss5 = SoftmaxWithLoss3D(pred5.output)
softmax_loss4 = SoftmaxWithLoss3D(pred4.output)
softmax_loss3 = SoftmaxWithLoss3D(pred3.output)
softmax_loss2 = SoftmaxWithLoss3D(pred2.output)
# self.loss = softmax_loss2.loss(self.y)
self.loss = (softmax_loss5.loss(label5) + softmax_loss4.loss(label4) +
softmax_loss3.loss(label3) + softmax_loss2.loss(self.y)) / 4.
# self.loss = self.loss /
self.error = softmax_loss2.error(self.y)
self.output = softmax_loss2.prediction()
self.params = get_trainable_params()
self.activations = []
def network_definition(self):
# (views, batch_size, 3, img_h, img_w)
self.x = tensor5()
self.is_x_tensor4 = False
img_w = self.img_w
img_h = self.img_h
n_gru_vox = [4, 8, 16, 32]
n_convfilter = [8, 16, 32, 64, 128]
n_fc_filters = [256]
n_deconvfilter = [128, 64, 32, 16, 2]
input_shape = (self.batch_size, 3, img_w, img_h)
fc_shape = (self.batch_size, n_fc_filters[0])
# To define the weights, define the net structure first
x = InputLayer(input_shape)
conv1a = ConvLayer(x, (n_convfilter[0], 7, 7))
conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3))
pool1 = PoolLayer(conv1b) # H/2
conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3))
conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3))
conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1))
pool2 = PoolLayer(conv2c) # H/4
conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3))
conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3))
conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1))
pool3 = PoolLayer(conv3c) # H/8
conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3))
conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3))
pool4 = PoolLayer(conv4b) # H/16
conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3))
conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3))
conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1)) # H/32
pool5 = PoolLayer(conv5b)
flat5 = FlattenLayer(pool5)
fc5 = TensorProductLayer(flat5, n_fc_filters[0])
flat4 = FlattenLayer(pool4)
fc4 = TensorProductLayer(flat4, n_fc_filters[0])
flat3 = FlattenLayer(pool3)
fc3 = TensorProductLayer(flat3, n_fc_filters[0])
flat2 = FlattenLayer(pool2)
fc2 = TensorProductLayer(flat2, n_fc_filters[0])
# flat1 = FlattenLayer(pool1)
# fc1 = TensorProductLayer(flat1, n_fc_filters[0])
# ==================== recurrence 5 ========================#
s_shape_5 = (self.batch_size, n_gru_vox[0], n_deconvfilter[0],
n_gru_vox[0], n_gru_vox[0])
# s_shape_5 = (self.batch_size, n_gru_vox[4], n_deconvfilter[4], n_gru_vox[4], n_gru_vox[4])
prev_s_5 = InputLayer(s_shape_5)
t_x_s_update_5 = FCConv3DLayer(
prev_s_5, fc5, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
t_x_s_reset_5 = FCConv3DLayer(
prev_s_5, fc5, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
reset_gate_5 = SigmoidLayer(t_x_s_reset_5)
rs_5 = EltwiseMultiplyLayer(reset_gate_5, prev_s_5)
t_x_rs_5 = FCConv3DLayer(
rs_5, fc5, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
# ==================== recurrence 4 ========================#
s_shape_4 = (self.batch_size, n_gru_vox[1], n_deconvfilter[1],
n_gru_vox[1], n_gru_vox[1])
prev_s_4 = InputLayer(s_shape_4)
t_x_s_update_4 = FCConv3DLayer(
prev_s_4, fc4, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3))
t_x_s_reset_4 = FCConv3DLayer(
prev_s_4, fc4, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3))
reset_gate_4 = SigmoidLayer(t_x_s_reset_4)
rs_4 = EltwiseMultiplyLayer(reset_gate_4, prev_s_4)
t_x_rs_4 = FCConv3DLayer(
rs_4, fc4, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3))
# =================== recurrence 3 =======================#
s_shape_3 = (self.batch_size, n_gru_vox[2], n_deconvfilter[2],
n_gru_vox[2], n_gru_vox[2])
prev_s_3 = InputLayer(s_shape_3)
t_x_s_update_3 = FCConv3DLayer(
prev_s_3, fc3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3))
t_x_s_reset_3 = FCConv3DLayer(
prev_s_3, fc3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3))
reset_gate_3 = SigmoidLayer(t_x_s_reset_3)
rs_3 = EltwiseMultiplyLayer(reset_gate_3, prev_s_3)
t_x_rs_3 = FCConv3DLayer(
rs_3, fc3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3))
# ================== recurrence 2 =======================#
s_shape_2 = (self.batch_size, n_gru_vox[3], n_deconvfilter[3],
n_gru_vox[3], n_gru_vox[3])
prev_s_2 = InputLayer(s_shape_2)
t_x_s_update_2 = FCConv3DLayer(
prev_s_2, fc2, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3))
t_x_s_reset_2 = FCConv3DLayer(
prev_s_2, fc2, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3))
reset_gate_2 = SigmoidLayer(t_x_s_reset_2)
rs_2 = EltwiseMultiplyLayer(reset_gate_2, prev_s_2)
t_x_rs_2 = FCConv3DLayer(
rs_2, fc2, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3))
# # ================= recurrence 1 ========================#
# s_shape_1 = (self.batch_size, n_gru_vox[4], n_deconvfilter[4], n_gru_vox[4], n_gru_vox[4])
# prev_s_1 = InputLayer(s_shape_1)
#
# t_x_s_update_1 = FCConv3DLayer(prev_s_1, fc1, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3))
# t_x_s_reset_1 = FCConv3DLayer(prev_s_1, fc1, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3))
#
# reset_gate_1 = SigmoidLayer(t_x_s_reset_1)
# rs_1 = EltwiseMultiplyLayer(reset_gate_1, prev_s_1)
# t_x_rs_1 = FCConv3DLayer(rs_1, fc1, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3))
def encode_recurrence(x_curr):
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3),
params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
# flat1_ = FlattenLayer(pool1_)
# fc1_ = TensorProductLayer(flat1_, n_fc_filters[0], params=fc1.params)
# out1_ = LeakyReLU(fc1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3),
params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3),
params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1),
params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
flat2_ = FlattenLayer(pool2_)
fc2_ = TensorProductLayer(flat2_,
n_fc_filters[0],
params=fc2.params)
out2_ = LeakyReLU(fc2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3),
params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3),
params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1),
params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
flat3_ = FlattenLayer(pool3_)
fc3_ = TensorProductLayer(flat3_,
n_fc_filters[0],
params=fc3.params)
out3_ = LeakyReLU(fc3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3),
params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3),
params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
flat4_ = FlattenLayer(pool4_)
fc4_ = TensorProductLayer(flat4_,
n_fc_filters[0],
params=fc4.params)
out4_ = LeakyReLU(fc4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3),
params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3),
params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1),
params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
flat5_ = FlattenLayer(pool5_)
fc5_ = TensorProductLayer(flat5_,
n_fc_filters[0],
params=fc5.params)
out5_ = LeakyReLU(fc5_)
return out5_.output, out4_.output, out3_.output, out2_.output # , out1_.output
s_encoder, _ = theano.scan(encode_recurrence, sequences=[self.x])
out_5 = s_encoder[0]
out_4 = s_encoder[1]
out_3 = s_encoder[2]
out_2 = s_encoder[3]
# out_1 = s_encoder[4]
def decode_recurrence_5(x_curr, prev_s_tensor, prev_in_gate_tensor):
x_curr_ = InputLayer(fc_shape, x_curr)
prev_s_5_ = InputLayer(s_shape_5, prev_s_tensor)
t_x_s_update_5_ = FCConv3DLayer(
prev_s_5_,
x_curr_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update_5.params)
t_x_s_reset_5_ = FCConv3DLayer(
prev_s_5_,
x_curr_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset_5.params)
update_gate_ = SigmoidLayer(t_x_s_update_5_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_5_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_5_)
t_x_rs_5_ = FCConv3DLayer(
rs_,
x_curr_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_rs_5.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_5_)
gru_out_5_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_5_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_5_.output, update_gate_.output
s_update_5_, _ = theano.scan(
decode_recurrence_5,
sequences=[out_5],
outputs_info=[
tensor.zeros_like(np.zeros(s_shape_5),
dtype=theano.config.floatX),
tensor.zeros_like(np.zeros(s_shape_5),
dtype=theano.config.floatX)
])
update_all_5 = s_update_5_[-1]
s_out_5 = update_all_5[0][-1]
input_5 = InputLayer(s_shape_5, s_out_5)
# Unpooling s_out_5
unpool5 = Unpool3DLayer(input_5)
conv_out5 = Conv3DLayer(unpool5, (64, 3, 3, 3))
print("conv_out5", conv_out5.output_shape)
def decode_recurrence_4(x_curr, prev_s_tensor, prev_in_gate_tensor):
x_curr_ = InputLayer(fc_shape, x_curr)
prev_s_4_ = InputLayer(s_shape_4, prev_s_tensor)
t_x_s_update_4_ = FCConv3DLayer(
prev_s_4_,
x_curr_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_s_update_4.params)
t_x_s_reset_4_ = FCConv3DLayer(
prev_s_4_,
x_curr_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_s_reset_4.params)
print("x_curr: ", x_curr_.output_shape)
print("prev_s_4_: ", prev_s_4_.output_shape)
print("t_x_s_update_4_: ", t_x_s_update_4_.output_shape)
print("t_x_s_reset_4_: ", t_x_s_reset_4_.output_shape)
update_gate_ = SigmoidLayer(t_x_s_update_4_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_4_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_4_)
t_x_rs_4_ = FCConv3DLayer(
rs_,
x_curr_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_rs_4.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_4_)
gru_out_4_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_4_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_4_.output, update_gate_.output
s_update_4_, _ = theano.scan(decode_recurrence_4,
sequences=[out_4],
outputs_info=[
conv_out5.output,
tensor.zeros_like(
np.zeros(s_shape_4),
dtype=theano.config.floatX)
])
update_all_4 = s_update_4_[-1]
s_out_4 = update_all_4[0][-1]
input_4 = InputLayer(s_shape_4, s_out_4)
# Unpooling s_out_4
unpool4 = Unpool3DLayer(input_4)
conv_out4 = Conv3DLayer(unpool4, (n_deconvfilter[2], 3, 3, 3))
print("conv_out_4: ", conv_out4.output_shape)
print("conv_out_4: ", conv_out4.output)
def decode_recurrence_3(x_curr, prev_s_tensor, prev_in_gate_tensor):
x_curr_ = InputLayer(fc_shape, x_curr)
prev_s_3_ = InputLayer(s_shape_3, prev_s_tensor)
t_x_s_update_3_ = FCConv3DLayer(
prev_s_3_,
x_curr_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_s_update_3.params)
t_x_s_reset_3_ = FCConv3DLayer(
prev_s_3_,
x_curr_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_s_reset_3.params)
update_gate_ = SigmoidLayer(t_x_s_update_3_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_3_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_3_)
t_x_rs_3_ = FCConv3DLayer(
rs_,
x_curr_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_rs_3.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_3_)
gru_out_3_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_3_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_3_.output, update_gate_.output
s_update_3_, _ = theano.scan(decode_recurrence_3,
sequences=[out_3],
outputs_info=[
conv_out4.output,
tensor.zeros_like(
np.zeros(s_shape_3),
dtype=theano.config.floatX)
])
update_all_3 = s_update_3_[-1]
s_out_3 = update_all_3[0][-1]
input_3 = InputLayer(s_shape_3, s_out_3)
# Unpooling s_out_4
unpool3 = Unpool3DLayer(input_3)
conv_out3 = Conv3DLayer(unpool3, (n_deconvfilter[3], 3, 3, 3))
print("conv_out_3: ", conv_out3.output_shape)
print("conv_out_3: ", conv_out3.output)
def decode_recurrence_2(x_curr, prev_s_tensor, prev_in_gate_tensor):
x_curr_ = InputLayer(fc_shape, x_curr)
prev_s_2_ = InputLayer(s_shape_2, prev_s_tensor)
t_x_s_update_2_ = FCConv3DLayer(
prev_s_2_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_update_2.params)
t_x_s_reset_2_ = FCConv3DLayer(
prev_s_2_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_reset_2.params)
update_gate_ = SigmoidLayer(t_x_s_update_2_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_2_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_2_)
t_x_rs_2_ = FCConv3DLayer(
rs_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_rs_2.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_2_)
gru_out_2_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_2_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_2_.output, update_gate_.output
s_update_2_, _ = theano.scan(decode_recurrence_2,
sequences=[out_2],
outputs_info=[
conv_out3.output,
tensor.zeros_like(
np.zeros(s_shape_2),
dtype=theano.config.floatX)
])
update_all_2 = s_update_2_[-1]
s_out_2 = update_all_2[0][-1]
input_2 = InputLayer(s_shape_2, s_out_2)
# Unpooling s_out_4
# unpool2 = Unpool3DLayer(input_2)
# conv_out2 = Unpool3DLayer(unpool2, (n_deconvfilter[4], 3, 3, 3))
# def decode_recurrence_1(x_curr, prev_s_tensor, prev_in_gate_tensor):
# x_curr_ = InputLayer(fc_shape, x_curr)
# prev_s_1_ = InputLayer(s_shape_1, prev_s_tensor)
# t_x_s_update_1_ = FCConv3DLayer(prev_s_1_,
# x_curr_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
# params=t_x_s_update_1.params)
#
# t_x_s_reset_1_ = FCConv3DLayer(prev_s_1_, x_curr_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
# params=t_x_s_reset_1.params)
#
# update_gate_ = SigmoidLayer(t_x_s_update_1_)
# comp_update_gate_ = ComplementLayer(update_gate_)
# reset_gate_ = SigmoidLayer(t_x_s_reset_1_)
#
# rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_1_)
# t_x_rs_1_ = FCConv3DLayer(rs_, x_curr_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
# params=t_x_rs_1.params)
# tanh_t_x_rs_ = TanhLayer(t_x_rs_1_)
#
# gru_out_1_ = AddLayer(
# EltwiseMultiplyLayer(update_gate_, prev_s_1_),
# EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
#
# return gru_out_1_.output, update_gate_.output
#
# s_update_1_, _ = theano.scan(decode_recurrence_1,
# sequences=[out_1],
# outputs_info=[conv_out2.output,
# tensor.zeros_like(np.zeros(s_shape_1),
# dtype=theano.config.floatX)])
# update_all_1 = s_update_1_[-1]
# s_out_1 = update_all_1[0][-1]
#
# s_out_1_input = InputLayer(s_shape_1, s_out_1)
conv_out2 = Conv3DLayer(input_2, (n_deconvfilter[4], 3, 3, 3))
softmax_loss = SoftmaxWithLoss3D(conv_out2.output)
print("conv_out_2: ", conv_out2.output_shape)
print("conv_out_2: ", conv_out2.output)
self.loss = softmax_loss.loss(self.y)
self.error = softmax_loss.error(self.y)
self.params = get_trainable_params()
self.output = softmax_loss.prediction()
self.activations = [
update_all_5, update_all_4, update_all_3, update_all_2
]
def network_definition(self):
# (multi_views, self.batch_size, 3, self.img_h, self.img_w),
self.x = tensor5()
self.is_x_tensor4 = False
img_w = self.img_w
img_h = self.img_h
n_gru_vox = 4
n_vox = self.n_vox
n_convfilter = [96, 128, 256, 256, 256, 256]
n_fc_filters = [1024, 2]
n_deconvfilter = [128, 128, 128, 128, 96, 2]
n_maskconvfilter = [96, 2]
n_conv_advfilter = [32, 128, 128, 128, 32]
n_fc_advfilter = [1024, 2]
input_shape = (self.batch_size, 3, img_w, img_h)
rendering_shape = (self.batch_size, n_deconvfilter[3], img_h, img_w)
voxel_shape = (self.batch_size, n_vox, n_vox, n_vox)
# To define weights, define the network structure first
x = InputLayer(input_shape)
conv1a = ConvLayer(x, (n_convfilter[0], 7, 7), param_type='generator')
conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3),
param_type='generator')
pool1 = PoolLayer(conv1b)
conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3),
param_type='generator')
conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3),
param_type='generator')
conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1),
param_type='generator')
pool2 = PoolLayer(conv2c)
conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3),
param_type='generator')
conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3),
param_type='generator')
conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1),
param_type='generator')
pool3 = PoolLayer(conv3b)
conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3),
param_type='generator')
conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3),
param_type='generator')
pool4 = PoolLayer(conv4b)
conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3),
param_type='generator')
conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3),
param_type='generator')
conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1),
param_type='generator')
pool5 = PoolLayer(conv5b)
conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3),
param_type='generator')
conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3),
param_type='generator')
pool6 = PoolLayer(conv6b)
flat6 = FlattenLayer(pool6)
fc7 = TensorProductLayer(flat6,
n_fc_filters[0],
param_type='generator')
# Set the size to be 256x4x4x4
s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox,
n_gru_vox)
# Dummy 3D grid hidden representations
prev_s = InputLayer(s_shape)
t_x_s_update = FCConv3DLayer(
prev_s,
fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
param_type='generator')
t_x_s_reset = FCConv3DLayer(
prev_s,
fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
param_type='generator')
reset_gate = SigmoidLayer(t_x_s_reset)
rs = EltwiseMultiplyLayer(reset_gate, prev_s)
t_x_rs = FCConv3DLayer(rs,
fc7,
(n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
param_type='generator')
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3),
params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3),
params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3),
params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1),
params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3),
params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3),
params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1),
params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3),
params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3),
params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3),
params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3),
params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1),
params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3),
params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3),
params=conv6b.params)
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
flat6_ = FlattenLayer(pool6_)
fc7_ = TensorProductLayer(flat6_,
n_fc_filters[0],
params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_.output, update_gate_.output
s_update, r_update = theano.scan(
recurrence,
sequences=[
self.x[:, :, :3]
], # along with images, feed in the index of the current frame
outputs_info=[
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX),
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX)
])
update_all = s_update[-1]
s_all = s_update[0]
s_last = s_all[-1]
gru_s = InputLayer(s_shape, s_last)
unpool7 = Unpool3DLayer(gru_s)
conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3),
param_type='generator')
rect7a = LeakyReLU(conv7a)
conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3),
param_type='generator')
rect7 = LeakyReLU(conv7b)
res7 = AddLayer(unpool7, rect7)
unpool8 = Unpool3DLayer(res7)
conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3),
param_type='generator')
rect8a = LeakyReLU(conv8a)
conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3),
param_type='generator')
rect8 = LeakyReLU(conv8b)
res8 = AddLayer(unpool8, rect8)
unpool9 = Unpool3DLayer(res8)
conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
rect9a = LeakyReLU(conv9a)
conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
rect9 = LeakyReLU(conv9b)
conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1),
param_type='generator')
res9 = AddLayer(conv9c, rect9)
conv10a = Conv3DLayer(res9, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
rect10a = LeakyReLU(conv10a)
conv10b = Conv3DLayer(rect10a, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
rect10 = LeakyReLU(conv10b)
conv10c = Conv3DLayer(rect10a, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
res10 = AddLayer(conv10c, rect10)
conv11 = Conv3DLayer(res10, (n_deconvfilter[4], 3, 3, 3),
param_type='generator')
conv12 = Conv3DLayer(conv11, (n_deconvfilter[5], 3, 3, 3),
param_type='generator')
voxel_loss = SoftmaxWithLoss3D(conv12.output)
reconstruction = voxel_loss.prediction()
voxel_input = InputLayer(voxel_shape, reconstruction[:, :, 1])
rend = RaytracingLayer(voxel_input, self.camera, img_w, img_h,
self.pad_x, self.pad_y)
assert not r_update, 'Unexpected update in the RNN.'
self.mask_loss = tensor.nnet.nnet.binary_crossentropy(
tensor.clip(rend.output[:, :, 0], 1e-7, 1.0 - 1e-7),
tensor.gt(self.x[:, :, 3],
0.).astype(theano.config.floatX)).mean()
self.voxel_loss = self.mask_loss
self.discriminator_loss = None
self.generator_loss = self.mask_loss
self.error = voxel_loss.error(self.y)
self.generator_params = get_trainable_params()['generator']
self.all_params = self.generator_params + self.discriminator_params
self.load_params = self.generator_params
self.output = reconstruction
self.activations = [rend.output[:, :, 0]]
def network_definition(self):
# (multi_views, self.batch_size, 3, self.img_h, self.img_w),
self.x = tensor5()
self.is_x_tensor4 = False
img_w = self.img_w
img_h = self.img_h
n_gru_vox = 4
# n_vox = self.n_vox
n_convfilter = [96, 128, 256, 256, 256, 256]
n_fc_filters = [1024]
n_deconvfilter = [128, 128, 128, 64, 32, 2]
input_shape = (self.batch_size, 3, img_w, img_h)
# To define weights, define the network structure first
x = InputLayer(input_shape)
conv1 = ConvLayer(x, (n_convfilter[0], 7, 7))
pool1 = PoolLayer(conv1)
conv2 = ConvLayer(pool1, (n_convfilter[1], 3, 3))
pool2 = PoolLayer(conv2)
conv3 = ConvLayer(pool2, (n_convfilter[2], 3, 3))
pool3 = PoolLayer(conv3)
conv4 = ConvLayer(pool3, (n_convfilter[3], 3, 3))
pool4 = PoolLayer(conv4)
conv5 = ConvLayer(pool4, (n_convfilter[4], 3, 3))
pool5 = PoolLayer(conv5)
conv6 = ConvLayer(pool5, (n_convfilter[5], 3, 3))
pool6 = PoolLayer(conv6)
flat6 = FlattenLayer(pool6)
fc7 = TensorProductLayer(flat6, n_fc_filters[0])
# Set the size to be 256x4x4x4
s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox,
n_gru_vox)
# Dummy 3D grid hidden representations
prev_s = InputLayer(s_shape)
t_x_s_update = FCConv3DLayer(
prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
t_x_s_reset = FCConv3DLayer(
prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
reset_gate = SigmoidLayer(t_x_s_reset)
rs = EltwiseMultiplyLayer(reset_gate, prev_s)
t_x_rs = FCConv3DLayer(rs, fc7,
(n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1.params)
pool1_ = PoolLayer(conv1_)
rect1_ = LeakyReLU(pool1_)
conv2_ = ConvLayer(rect1_, (n_convfilter[1], 3, 3),
params=conv2.params)
pool2_ = PoolLayer(conv2_)
rect2_ = LeakyReLU(pool2_)
conv3_ = ConvLayer(rect2_, (n_convfilter[2], 3, 3),
params=conv3.params)
pool3_ = PoolLayer(conv3_)
rect3_ = LeakyReLU(pool3_)
conv4_ = ConvLayer(rect3_, (n_convfilter[3], 3, 3),
params=conv4.params)
pool4_ = PoolLayer(conv4_)
rect4_ = LeakyReLU(pool4_)
conv5_ = ConvLayer(rect4_, (n_convfilter[4], 3, 3),
params=conv5.params)
pool5_ = PoolLayer(conv5_)
rect5_ = LeakyReLU(pool5_)
conv6_ = ConvLayer(rect5_, (n_convfilter[5], 3, 3),
params=conv6.params)
pool6_ = PoolLayer(conv6_)
rect6_ = LeakyReLU(pool6_)
flat6_ = FlattenLayer(rect6_)
fc7_ = TensorProductLayer(flat6_,
n_fc_filters[0],
params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_.output, update_gate_.output
s_update, _ = theano.scan(
recurrence,
sequences=[
self.x
], # along with images, feed in the index of the current frame
outputs_info=[
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX),
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX)
])
update_all = s_update[-1]
s_all = s_update[0]
s_last = s_all[-1]
gru_s = InputLayer(s_shape, s_last)
unpool7 = Unpool3DLayer(gru_s)
conv7 = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3))
rect7 = LeakyReLU(conv7)
unpool8 = Unpool3DLayer(rect7)
conv8 = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3))
rect8 = LeakyReLU(conv8)
unpool9 = Unpool3DLayer(rect8)
conv9 = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3))
rect9 = LeakyReLU(conv9)
# unpool10 = Unpool3DLayer(rect9)
conv10 = Conv3DLayer(rect9, (n_deconvfilter[4], 3, 3, 3))
rect10 = LeakyReLU(conv10)
conv11 = Conv3DLayer(rect10, (n_deconvfilter[5], 3, 3, 3))
softmax_loss = SoftmaxWithLoss3D(conv11.output)
self.loss = softmax_loss.loss(self.y)
self.error = softmax_loss.error(self.y)
self.params = get_trainable_params()
self.output = softmax_loss.prediction()
self.activations = [update_all]
def network_definition(self):
# (multi_views, self.batch_size, 3, self.img_h, self.img_w),
self.x = tensor5()
self.is_x_tensor4 = False
img_w = self.img_w
img_h = self.img_h
n_gru_vox = 4
n_vox = self.n_vox
n_convfilter = [96, 128, 256, 256, 256, 256]
n_fc_filters = [1024, 2]
n_deconvfilter = [128, 128, 128, 128, 96, 2]
n_conv_advfilter = [32, 128, 128, 128, 32]
n_fc_advfilter = [1024, 2]
input_shape = (self.batch_size, 3, img_w, img_h)
voxel_shape = (self.batch_size, n_vox, n_vox, n_vox)
# To define weights, define the network structure first
x = InputLayer(input_shape)
conv1a = ConvLayer(x, (n_convfilter[0], 7, 7), param_type='generator')
conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3),
param_type='generator')
pool1 = PoolLayer(conv1b)
conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3),
param_type='generator')
conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3),
param_type='generator')
conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1),
param_type='generator')
pool2 = PoolLayer(conv2c)
conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3),
param_type='generator')
conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3),
param_type='generator')
conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1),
param_type='generator')
pool3 = PoolLayer(conv3b)
conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3),
param_type='generator')
conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3),
param_type='generator')
pool4 = PoolLayer(conv4b)
conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3),
param_type='generator')
conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3),
param_type='generator')
conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1),
param_type='generator')
pool5 = PoolLayer(conv5b)
conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3),
param_type='generator')
conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3),
param_type='generator')
pool6 = PoolLayer(conv6b)
flat6 = FlattenLayer(pool6)
fc7 = TensorProductLayer(flat6,
n_fc_filters[0],
param_type='generator')
# Set the size to be 256x4x4x4
s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox,
n_gru_vox)
# Dummy 3D grid hidden representations
prev_s = InputLayer(s_shape)
t_x_s_update = FCConv3DLayer(
prev_s,
fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
param_type='generator')
t_x_s_reset = FCConv3DLayer(
prev_s,
fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
param_type='generator')
reset_gate = SigmoidLayer(t_x_s_reset)
rs = EltwiseMultiplyLayer(reset_gate, prev_s)
t_x_rs = FCConv3DLayer(rs,
fc7,
(n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
param_type='generator')
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3),
params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3),
params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3),
params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1),
params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3),
params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3),
params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1),
params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3),
params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3),
params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3),
params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3),
params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1),
params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3),
params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3),
params=conv6b.params)
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
flat6_ = FlattenLayer(pool6_)
fc7_ = TensorProductLayer(flat6_,
n_fc_filters[0],
params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_.output, update_gate_.output
s_update, r_update = theano.scan(
recurrence,
sequences=[
self.x[:, :, :3]
], # along with images, feed in the index of the current frame
outputs_info=[
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX),
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX)
])
update_all = s_update[-1]
s_all = s_update[0]
s_last = s_all[-1]
gru_s = InputLayer(s_shape, s_last)
unpool7 = Unpool3DLayer(gru_s)
conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3),
param_type='generator')
rect7a = LeakyReLU(conv7a)
conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3),
param_type='generator')
rect7 = LeakyReLU(conv7b)
res7 = AddLayer(unpool7, rect7)
unpool8 = Unpool3DLayer(res7)
conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3),
param_type='generator')
rect8a = LeakyReLU(conv8a)
conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3),
param_type='generator')
rect8 = LeakyReLU(conv8b)
res8 = AddLayer(unpool8, rect8)
unpool9 = Unpool3DLayer(res8)
conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
rect9a = LeakyReLU(conv9a)
conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
rect9 = LeakyReLU(conv9b)
conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1),
param_type='generator')
res9 = AddLayer(conv9c, rect9)
conv10a = Conv3DLayer(res9, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
rect10a = LeakyReLU(conv10a)
conv10b = Conv3DLayer(rect10a, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
rect10 = LeakyReLU(conv10b)
conv10c = Conv3DLayer(rect10a, (n_deconvfilter[3], 3, 3, 3),
param_type='generator')
res10 = AddLayer(conv10c, rect10)
conv11 = Conv3DLayer(res10, (n_deconvfilter[4], 3, 3, 3),
param_type='generator')
conv12 = Conv3DLayer(conv11, (n_deconvfilter[5], 3, 3, 3),
param_type='generator')
voxel_loss = SoftmaxWithLoss3D(conv12.output)
reconstruction = voxel_loss.prediction()
voxel_input = InputLayer(voxel_shape, reconstruction[:, :, 1])
rend = RaytracingLayer(voxel_input, self.camera, img_w, img_h,
self.pad_x, self.pad_y)
# Discriminator network starts here.
disc_input = InputLayer(voxel_shape)
disc_padded = DimShuffleLayer(disc_input, (0, 1, 'x', 2, 3))
conv15 = Conv3DLayer(disc_padded, (n_conv_advfilter[0], 3, 3, 3),
param_type='discriminator')
conv16 = Conv3DLayer(conv15, (n_conv_advfilter[0], 3, 3, 3),
param_type='discriminator')
pool16 = Pool3DLayer(conv16) # b x 16 x c x 16 x 16
conv17 = Conv3DLayer(pool16, (n_conv_advfilter[1], 3, 3, 3),
param_type='discriminator')
conv18 = Conv3DLayer(conv17, (n_conv_advfilter[1], 3, 3, 3),
param_type='discriminator')
pool18 = Pool3DLayer(conv18) # b x 8 x c x 8 x 8
conv19 = Conv3DLayer(pool18, (n_conv_advfilter[2], 3, 3, 3),
param_type='discriminator')
conv20 = Conv3DLayer(conv19, (n_conv_advfilter[2], 3, 3, 3),
param_type='discriminator')
pool20 = Pool3DLayer(conv20) # b x 4 x c x 4 x 4
conv21 = Conv3DLayer(pool20, (n_conv_advfilter[3], 3, 3, 3),
param_type='discriminator')
conv22 = Conv3DLayer(conv21, (n_conv_advfilter[3], 3, 3, 3),
param_type='discriminator')
pool22 = Pool3DLayer(conv22) # b x 2 x c x 2 x 2
conv23 = Conv3DLayer(pool22, (n_conv_advfilter[4], 3, 3, 3),
param_type='discriminator')
conv24 = Conv3DLayer(conv23, (n_conv_advfilter[4], 1, 1, 1),
param_type='discriminator')
flat24 = FlattenLayer(conv24)
fc24 = TensorProductLayer(flat24,
n_fc_advfilter[1],
param_type='discriminator')
def get_discriminator(data_centered, use_dropout):
conv15_ = Conv3DLayer(data_centered,
(n_conv_advfilter[0], 3, 3, 3),
params=conv15.params)
rect15_ = LeakyReLU(conv15_)
conv16_ = Conv3DLayer(rect15_, (n_conv_advfilter[0], 3, 3, 3),
params=conv16.params)
rect16_ = LeakyReLU(conv16_)
pool16_ = Pool3DLayer(rect16_) # b x 16 x c x 16 x 16
conv17_ = Conv3DLayer(pool16_, (n_conv_advfilter[1], 3, 3, 3),
params=conv17.params)
rect17_ = LeakyReLU(conv17_)
conv18_ = Conv3DLayer(rect17_, (n_conv_advfilter[1], 3, 3, 3),
params=conv18.params)
rect18_ = LeakyReLU(conv18_)
pool18_ = Pool3DLayer(rect18_) # b x 8 x c x 8 x 8
conv19_ = Conv3DLayer(pool18_, (n_conv_advfilter[2], 3, 3, 3),
params=conv19.params)
rect19_ = LeakyReLU(conv19_)
conv20_ = Conv3DLayer(rect19_, (n_conv_advfilter[2], 3, 3, 3),
params=conv20.params)
rect20_ = LeakyReLU(conv20_)
pool20_ = Pool3DLayer(rect20_) # b x 4 x c x 4 x 4
conv21_ = Conv3DLayer(pool20_, (n_conv_advfilter[3], 3, 3, 3),
params=conv21.params)
rect21_ = LeakyReLU(conv21_)
conv22_ = Conv3DLayer(rect21_, (n_conv_advfilter[3], 3, 3, 3),
params=conv22.params)
rect22_ = LeakyReLU(conv22_)
pool22_ = Pool3DLayer(rect22_) # b x 2 x c x 2 x 2
conv23_ = Conv3DLayer(pool22_, (n_conv_advfilter[4], 3, 3, 3),
params=conv23.params)
rect23_ = LeakyReLU(conv23_)
conv24_ = Conv3DLayer(rect23_, (n_conv_advfilter[4], 1, 1, 1),
params=conv24.params)
flat24_ = FlattenLayer(conv24_)
fc24_ = TensorProductLayer(flat24_,
n_fc_advfilter[1],
params=fc24.params)
return SoftmaxWithLoss3D(fc24_.output, axis=1)
voxel_padded = DimShuffleLayer(voxel_input, (0, 1, 'x', 2, 3))
if cfg.TRAIN.STABILIZER == 'diffstep':
voxel_stabilized = DifferentiableStepLayer(
voxel_padded, backprop=cfg.TRAIN.DIFF_BACKPROP)
elif cfg.TRAIN.STABILIZER == 'noise':
voxel_stabilized = InstanceNoiseLayer(voxel_padded,
std=self.noise *
cfg.TRAIN.NOISE_MAXSTD)
elif cfg.TRAIN.STABILIZER == 'ignore':
voxel_stabilized = voxel_padded
else:
raise NotImplemented
voxel_centered = SubtractLayer(voxel_stabilized, 0.5)
gt_input = InputLayer(voxel_shape, self.y[:, :, 1])
gt_padded = DimShuffleLayer(gt_input, (0, 1, 'x', 2, 3))
if cfg.TRAIN.STABILIZER == 'diffstep':
gt_stabilized = gt_padded
elif cfg.TRAIN.STABILIZER == 'noise':
gt_stabilized = InstanceNoiseLayer(gt_padded,
std=self.noise *
cfg.TRAIN.NOISE_MAXSTD)
elif cfg.TRAIN.STABILIZER == 'ignore':
gt_stabilized = gt_padded
else:
raise NotImplemented
gt_centered = SubtractLayer(gt_stabilized, 0.5)
# Discriminator 1: takes fake voxel as input.
discriminator_fake_loss = get_discriminator(voxel_centered, True)
# Discriminator 2: takes real voxel as input.
discriminator_real_loss = get_discriminator(gt_centered, True)
# Discriminator 3: takes generated voxel as input, doesn't use dropout.
discriminator_fake_test = get_discriminator(voxel_centered, False)
# Discriminator 4: takes real voxel as input, doesn't use dropout.
discriminator_real_test = get_discriminator(gt_centered, False)
assert not r_update, 'Unexpected update in the RNN.'
label_shape = np.zeros((self.batch_size, 1))
fake_label = tensor.zeros_like(label_shape, dtype=theano.config.floatX)
real_label = tensor.ones_like(label_shape, dtype=theano.config.floatX)
all_fake = tensor.concatenate((real_label, fake_label), axis=1)
all_real = tensor.concatenate((fake_label, real_label), axis=1)
self.voxel_loss = discriminator_fake_test.loss(all_real)
self.mask_loss = tensor.nnet.nnet.binary_crossentropy(
tensor.clip(rend.output[:, :, 0], 1e-7, 1.0 - 1e-7),
tensor.gt(self.x[:, :, 3],
0.).astype(theano.config.floatX)).mean()
self.discriminator_loss = (discriminator_fake_loss.loss(all_fake) +
discriminator_real_loss.loss(all_real)) / 2.
self.generator_loss = self.voxel_loss + self.mask_loss * 100
self.error = voxel_loss.error(self.y)
self.error_F = discriminator_fake_test.error(all_fake)
self.error_R = discriminator_real_test.error(all_real)
self.generator_params = get_trainable_params()['generator']
self.discriminator_params = get_trainable_params()['discriminator']
self.all_params = self.generator_params + self.discriminator_params
self.load_params = self.all_params
self.output = reconstruction
self.activations = [rend.output[:, :, 0]]
def network_definition(self):
# (multi_views, self.batch_size, 3, self.img_h, self.img_w),
self.x = tensor5()
self.is_x_tensor4 = False
img_w = self.img_w
img_h = self.img_h
n_gru_vox = 4
# n_vox = self.n_vox
n_convfilter = [96, 128, 256, 256, 256, 256]
n_fc_filters = [1024]
n_deconvfilter = [128, 128, 128, 64, 32, 2]
input_shape = (self.batch_size, 3, img_w, img_h)
# To define weights, define the network structure first
x = InputLayer(input_shape)
conv1a = ConvLayer(x, (n_convfilter[0], 7, 7))
conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3))
pool1 = PoolLayer(conv1b)
conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3))
conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3))
conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1))
pool2 = PoolLayer(conv2c)
conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3))
conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3))
conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1))
pool3 = PoolLayer(conv3b)
conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3))
conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3))
pool4 = PoolLayer(conv4b)
conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3))
conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3))
conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1))
pool5 = PoolLayer(conv5b)
conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3))
conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3))
pool6 = PoolLayer(conv6b)
flat6 = FlattenLayer(pool6)
fc7 = TensorProductLayer(flat6, n_fc_filters[0])
#LSTM
# Set the size to be 256x4x4x4
h_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox,
n_gru_vox)
# Dummy 3D grid hidden representations
prev_h = InputLayer(h_shape)
t_x_s_forget = FCConv3DLayer(
prev_h, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
t_x_s_input = FCConv3DLayer(
prev_h, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
t_x_s_cell = FCConv3DLayer(
prev_h, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
#initialize hidden state and cell state with 0
if self.hidden_last is None:
self.hidden_last = theano.shared(
np.zeros(h_shape, dtype=theano.config.floatX))
if self.cell_last is None:
self.cell_last = theano.shared(
np.zeros(h_shape, dtype=theano.config.floatX))
def recurrence(x_curr, prev_h_tensor, prev_s_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3),
params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3),
params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3),
params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1),
params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3),
params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3),
params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1),
params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3),
params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3),
params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3),
params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3),
params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1),
params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3),
params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3),
params=conv6b.params)
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
flat6_ = FlattenLayer(pool6_)
fc7_ = TensorProductLayer(flat6_,
n_fc_filters[0],
params=fc7.params)
rect7_ = LeakyReLU(fc7_)
#LSTM
# Dummy 3D grid hidden representations for previous hidden state and cell state
prev_h_ = InputLayer(h_shape, prev_h_tensor)
prev_s_ = InputLayer(h_shape, prev_s_tensor)
t_x_s_forget_ = FCConv3DLayer(
prev_h_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_forget.params)
t_x_s_input_ = FCConv3DLayer(
prev_h_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_input.params)
t_x_s_cell_ = FCConv3DLayer(
prev_h_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_cell.params)
forget_gate_ = SigmoidLayer(t_x_s_forget_)
input_gate_ = SigmoidLayer(t_x_s_input_)
tanh_t_x_s_cell_ = TanhLayer(t_x_s_cell_)
#current cell state
cell_state_ = AddLayer(
EltwiseMultiplyLayer(forget_gate_, prev_s_),
EltwiseMultiplyLayer(input_gate_, tanh_t_x_s_cell_))
#current hidden state, i.e. the output of lstm
hidden_state_ = TanhLayer(cell_state_)
return hidden_state_.output, cell_state_.output
s_update, _ = theano.scan(
recurrence,
sequences=[
self.x
], # along with images, feed in the index of the current frame
outputs_info=[
self.hidden_last.get_value(),
self.cell_last.get_value()
])
#s_update means updates of hidden states and cell states
cell_all = s_update[-1]
h_all = s_update[0]
h_last = h_all[-1]
lstm_s = InputLayer(h_shape, h_last)
unpool7 = Unpool3DLayer(lstm_s)
conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3))
rect7a = LeakyReLU(conv7a)
conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3))
rect7 = LeakyReLU(conv7b)
res7 = AddLayer(unpool7, rect7)
unpool8 = Unpool3DLayer(res7)
conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3))
rect8a = LeakyReLU(conv8a)
conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3))
rect8 = LeakyReLU(conv8b)
res8 = AddLayer(unpool8, rect8)
unpool9 = Unpool3DLayer(res8)
conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3))
rect9a = LeakyReLU(conv9a)
conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3))
rect9 = LeakyReLU(conv9b)
conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1))
res9 = AddLayer(conv9c, rect9)
conv10a = Conv3DLayer(res9, (n_deconvfilter[4], 3, 3, 3))
rect10a = LeakyReLU(conv10a)
conv10b = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3))
rect10 = LeakyReLU(conv10b)
conv10c = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3))
res10 = AddLayer(conv10c, rect10)
conv11 = Conv3DLayer(res10, (n_deconvfilter[5], 3, 3, 3))
softmax_loss = SoftmaxWithLoss3D(conv11.output)
self.loss = softmax_loss.loss(self.y)
self.error = softmax_loss.error(self.y)
self.params = get_trainable_params()
self.output = softmax_loss.prediction()
self.activations = [cell_all]
self.new_hidden_last = h_last
self.new_cell_last = cell_all[-1]
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Input layer
input_ = InputLayer(input_shape, x_curr)
# GRU network same parameters as encoder
# Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv1a_ = ConvLayer(input_, (gru_filters[0], 7, 7), params=conv1a.params) # 96 x 7 x 7
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (gru_filters[0], 3, 3), params=conv1b.params) # 96 x 3 x 3
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv2a_ = ConvLayer(pool1_, (gru_filters[1], 3, 3), params=conv2a.params) # 128 x 3 x 3
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (gru_filters[1], 3, 3), params=conv2b.params) # 128 x 3 x 3
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (gru_filters[1], 1, 1), params=conv2c.params) # 128 x 1 x 1
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv3a_ = ConvLayer(pool2_, (gru_filters[2], 3, 3), params=conv3a.params) # 256 x 3 x 3
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (gru_filters[2], 3, 3), params=conv3b.params) # 256 x 3 x 3
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (gru_filters[2], 1, 1), params=conv3c.params) # 256 x 1 x 1
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
# Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv4a_ = ConvLayer(pool3_, (gru_filters[3], 3, 3), params=conv4a.params) # 256 x 3 x 3
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (gru_filters[3], 3, 3), params=conv4b.params) # 256 x 3 x 3
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv5a_ = ConvLayer(pool4_, (gru_filters[4], 3, 3), params=conv5a.params) # 256 x 3 x 3
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (gru_filters[4], 3, 3), params=conv5b.params) # 256 x 3 x 3
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (gru_filters[4], 1, 1), params=conv5c.params) # 256 x 1 x 1
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv6a_ = ConvLayer(pool5_, (gru_filters[5], 3, 3), params=conv6a.params) # 256 x 3 x 3
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (gru_filters[5], 3, 3), params=conv6b.params) # 256 x 3 x 3
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
# Flatten Layer
flat6_ = FlattenLayer(pool6_)
# Fully connected layer
fc7_ = TensorProductLayer(flat6_, fully_connecter_filter[0], params=fc7.params)
rect7_ = LeakyReLU(fc7_)
# h(t-1)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
# FC layer convoluted with hidden states
update_layer_ = FCConv3DLayer(
prev_s_,
rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3
params=update_layer.params)
# FC layer convoluted with hidden states
reset_layer_ = FCConv3DLayer(
prev_s_,
rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3
params=reset_layer.params)
# Sigmoid( Wfx T(xt) (+) Uf * h(t-1) + bf )
update_gate_ = SigmoidLayer(update_layer_)
# 1 - u(t)
compliment_update_gate_ = ComplementLayer(update_gate_)
# Sigmoid (Wix T(xt) (+) Ui * h(t-1) + bi)
reset_gate_ = SigmoidLayer(reset_layer_)
# rt (.) h(t-1)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
# Uh * rt (.) h(t-1) + bh
hidden_layer_ = FCConv3DLayer(
rs_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), params=hidden_state_layer.params) # 128 x 3 x 3 x 3
tanh_layer = TanhLayer(hidden_layer_)
# ht = (1 - ut) (.) h(t-1) (+) tanh( Uh * rt (.) h(t-1) + bh )
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(compliment_update_gate_, tanh_layer))
return gru_out_.output, update_gate_.output
def network_definition(self):
# Depth of the convolutional layers. VGG Style
cnn_filters = [96, 128, 256, 256, 256, 256]
# One fully connected layer for a 1024 feature vector
fully_connecter_filter = [1024]
# Shape of input layers. Used by encoder and GRU
input_shape = (self.batch_size, 3, self.img_width, self.img_height)
######### Encoder ##########
# Input Layer
x = InputLayer(input_shape)
## First set of convolutional layers ##
conv1a = ConvLayer(x, (cnn_filters[0], 7, 7)) # 96 x 7 x 7
conv1b = ConvLayer(conv1a, (cnn_filters[0], 3, 3)) # 96 x 3 x 3
pool1 = PoolLayer(conv1b) # Max Pooling
## Second set of convolutional layers ##
conv2a = ConvLayer(pool1, (cnn_filters[1], 3, 3)) # 128 x 3 x 3
conv2b = ConvLayer(conv2a, (cnn_filters[1], 3, 3)) # 128 x 3 x 3
conv2c = ConvLayer(conv2b, (cnn_filters[1], 1, 1)) # 128 x 1 x 1
pool2 = PoolLayer(conv2c) # Max Pooling
## Third set of convolutional layers ##
conv3a = ConvLayer(pool2, (cnn_filters[2], 3, 3)) # 256 x 3 x 3
conv3b = ConvLayer(conv3a, (cnn_filters[2], 3, 3)) # 256 x 3 x 3
conv3c = ConvLayer(pool2, (cnn_filters[2], 1, 1)) # 256 x 1 x 1
pool3 = PoolLayer(conv3b) # Max Pooling
## Fourth set of convolutional layers ##
conv4a = ConvLayer(pool3, (cnn_filters[3], 3, 3)) # 256 x 3 x 3
conv4b = ConvLayer(conv4a, (cnn_filters[3], 3, 3)) # 256 x 3 x 3
pool4 = PoolLayer(conv4b) # Max Pooling
## Fifth set of convolutional layers ##
conv5a = ConvLayer(pool4, (cnn_filters[4], 3, 3)) # 256 x 3 x 3
conv5b = ConvLayer(conv5a, (cnn_filters[4], 3, 3)) # 256 x 3 x 3
conv5c = ConvLayer(pool4, (cnn_filters[4], 1, 1)) # 256 x 1 x 1
pool5 = PoolLayer(conv5b) # Max pooling
## Sixth set of convolutional layers ##
conv6a = ConvLayer(pool5, (cnn_filters[5], 3, 3)) # 256 x 3 x 3
conv6b = ConvLayer(conv6a, (cnn_filters[5], 3, 3)) # 256 x 3 x 3
pool6 = PoolLayer(conv6b)
# Flatten layer
flat6 = FlattenLayer(pool6)
# Fully Connected layer
fc7 = TensorProductLayer(flat6, 1024) # 1024 feature vector
########## End Encoder ############
########## Gated Recurrent Unit ############
# Filter size of layers within the unit
gru_filters = [96, 128, 256, 256, 256, 256]
# The 3D Convolutional LSTM has a grid structure of 4 x 4 x 4. 128 for first layer of decoder
s_shape = (self.batch_size, self.n_gru_vox, gru_filters[1], self.n_gru_vox, self.n_gru_vox)
# Initialize the first previous state to nothing
prev_s = InputLayer(s_shape) # h(t-1)
# 3 x 3 x 3 Convolution of hidden states of self and neighbors
# Wfx T(xt) (+) Uf * h(t-1) + bf
update_layer = FCConv3DLayer(prev_s, fc7, (gru_filters[1], gru_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3
# Wix T(xt) (+) Ui * h(t-1) + bi
reset_layer = FCConv3DLayer(prev_s, fc7, (gru_filters[1], gru_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3
# Sigmoid (Wix T(xt) (+) Ui * h(t-1) + bi)
reset_gate = SigmoidLayer(reset_layer)
# rt (.) h(t-1)
rs = EltwiseMultiplyLayer(reset_gate, prev_s) # Used for h(t)
# Wh T(xt) (+) Uh * (rt (.) h(t-1) + bh
hidden_state_layer = FCConv3DLayer(rs, fc7, (gru_filters[1], gru_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3
# Recurrence unit
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Input layer
input_ = InputLayer(input_shape, x_curr)
# GRU network same parameters as encoder
# Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv1a_ = ConvLayer(input_, (gru_filters[0], 7, 7), params=conv1a.params) # 96 x 7 x 7
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (gru_filters[0], 3, 3), params=conv1b.params) # 96 x 3 x 3
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv2a_ = ConvLayer(pool1_, (gru_filters[1], 3, 3), params=conv2a.params) # 128 x 3 x 3
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (gru_filters[1], 3, 3), params=conv2b.params) # 128 x 3 x 3
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (gru_filters[1], 1, 1), params=conv2c.params) # 128 x 1 x 1
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv3a_ = ConvLayer(pool2_, (gru_filters[2], 3, 3), params=conv3a.params) # 256 x 3 x 3
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (gru_filters[2], 3, 3), params=conv3b.params) # 256 x 3 x 3
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (gru_filters[2], 1, 1), params=conv3c.params) # 256 x 1 x 1
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
# Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv4a_ = ConvLayer(pool3_, (gru_filters[3], 3, 3), params=conv4a.params) # 256 x 3 x 3
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (gru_filters[3], 3, 3), params=conv4b.params) # 256 x 3 x 3
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv5a_ = ConvLayer(pool4_, (gru_filters[4], 3, 3), params=conv5a.params) # 256 x 3 x 3
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (gru_filters[4], 3, 3), params=conv5b.params) # 256 x 3 x 3
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (gru_filters[4], 1, 1), params=conv5c.params) # 256 x 1 x 1
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv6a_ = ConvLayer(pool5_, (gru_filters[5], 3, 3), params=conv6a.params) # 256 x 3 x 3
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (gru_filters[5], 3, 3), params=conv6b.params) # 256 x 3 x 3
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
# Flatten Layer
flat6_ = FlattenLayer(pool6_)
# Fully connected layer
fc7_ = TensorProductLayer(flat6_, fully_connecter_filter[0], params=fc7.params)
rect7_ = LeakyReLU(fc7_)
# h(t-1)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
# FC layer convoluted with hidden states
update_layer_ = FCConv3DLayer(
prev_s_,
rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3
params=update_layer.params)
# FC layer convoluted with hidden states
reset_layer_ = FCConv3DLayer(
prev_s_,
rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3
params=reset_layer.params)
# Sigmoid( Wfx T(xt) (+) Uf * h(t-1) + bf )
update_gate_ = SigmoidLayer(update_layer_)
# 1 - u(t)
compliment_update_gate_ = ComplementLayer(update_gate_)
# Sigmoid (Wix T(xt) (+) Ui * h(t-1) + bi)
reset_gate_ = SigmoidLayer(reset_layer_)
# rt (.) h(t-1)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
# Uh * rt (.) h(t-1) + bh
hidden_layer_ = FCConv3DLayer(
rs_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), params=hidden_state_layer.params) # 128 x 3 x 3 x 3
tanh_layer = TanhLayer(hidden_layer_)
# ht = (1 - ut) (.) h(t-1) (+) tanh( Uh * rt (.) h(t-1) + bh )
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(compliment_update_gate_, tanh_layer))
return gru_out_.output, update_gate_.output
s_update, _ = theano.scan(recurrence,
sequences=[self.x], # along with images, feed in the index of the current frame
outputs_info=[tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX),
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX)])
# Update of all units
update_all = s_update[-1]
s_all = s_update[0]
# Last hidden states. last timestep
s_last = s_all[-1]
########## End GRU ##########
########## Decoder ##########
# Depth of deconvolutional layers
dcnn_filters = [128, 128, 128, 64, 32, 2]
# Input Layer
gru_s = InputLayer(s_shape, s_last)
# Residual |=> ----------------------------------------------=V
# Unpooling -> deconvolution -> LeakyReLU -> DeConv -> LeakyReLU ->
unpool7 = Unpool3DLayer(gru_s)
conv7a = Conv3DLayer(unpool7, (dcnn_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3
rect7a = LeakyReLU(conv7a)
conv7b = Conv3DLayer(rect7a, (dcnn_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3
rect7 = LeakyReLU(conv7b)
res7 = AddLayer(unpool7, rect7)
# Residual |=> ----------------------------------------------=V
# Unpooling -> deconvolution -> LeakyReLU -> DeConv -> LeakyReLU ->
unpool8 = Unpool3DLayer(res7)
conv8a = Conv3DLayer(unpool8, (dcnn_filters[2], 3, 3, 3)) # 128 x 3 x 3 x 3
rect8a = LeakyReLU(conv8a)
conv8b = Conv3DLayer(rect8a, (dcnn_filters[2], 3, 3, 3)) # 128 x 3 x 3 x 3
rect8 = LeakyReLU(conv8b)
res8 = AddLayer(unpool8, rect8)
# Residual |=> ----------------------------------------------=V
# Unpooling -> deconvolution -> LeakyReLU -> DeConv -> LeakyReLU ->
unpool9 = Unpool3DLayer(res8)
conv9a = Conv3DLayer(unpool9, (dcnn_filters[3], 3, 3, 3)) # 64 x 3 x 3 x 3
rect9a = LeakyReLU(conv9a)
conv9b = Conv3DLayer(rect9a, (dcnn_filters[3], 3, 3, 3)) # 64 x 3 x 3 x 3
rect9 = LeakyReLU(conv9b)
conv9c = Conv3DLayer(unpool9, (dcnn_filters[3], 1, 1, 1)) # 64 x 1 x 1 x 1
res9 = AddLayer(conv9c, rect9)
# Residual |=> ----------------------------------------------=V
# Unpooling -> deconvolution -> LeakyReLU -> DeConv -> LeakyReLU ->
conv10a = Conv3DLayer(res9, (dcnn_filters[4], 3, 3, 3)) # 32 x 3 x 3 x 3
rect10a = LeakyReLU(conv10a)
conv10b = Conv3DLayer(rect10a, (dcnn_filters[4], 3, 3, 3)) # 32 x 3 x 3 x 3
rect10 = LeakyReLU(conv10b)
conv10c = Conv3DLayer(rect10a, (dcnn_filters[4], 3, 3, 3)) # 32 x 3 x 3 x 3
res10 = AddLayer(conv10c, rect10)
# Last convolution
conv11 = Conv3DLayer(res10, (dcnn_filters[5], 3, 3, 3)) # 2 x 3 x 3 x 3
# Softmax layer
softmax_loss = SoftmaxWithLoss3D(conv11.output)
########## End Decoder #########
self.loss = softmax_loss.loss(self.y)
self.error = softmax_loss.error(self.y)
self.params = get_trainable_params()
self.output = softmax_loss.prediction()
self.activations = [update_all]
def network_definition(self):
# theano.config.compute_test_value = 'warn'
self.x = tensor5()
self.is_x_tensor4 = False
img_w = self.img_w
img_h = self.img_h
n_gru_vox = 4
# n_vox = self.n_vox
n_convfilter = [96, 128, 256, 256, 256, 256]
n_fc_filters = [1024]
n_deconvfilter = [128, 128, 128, 64, 32, 2]
input_shape = (self.batch_size, 3, img_w, img_h)
# To define weights, define the network structure first
x = InputLayer(input_shape)
conv1a = ConvLayer(x, (n_convfilter[0], 7, 7))
conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3))
pool1 = PoolLayer(conv1b)
conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3))
conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3))
conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1))
pool2 = PoolLayer(conv2c)
conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3))
conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3))
conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1))
pool3 = PoolLayer(conv3b)
conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3))
conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3))
pool4 = PoolLayer(conv4b)
conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3))
conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3))
conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1))
pool5 = PoolLayer(conv5b)
conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3))
conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3))
pool6 = PoolLayer(conv6b)
flat6 = FlattenLayer(pool6)
fc7 = TensorProductLayer(flat6, n_fc_filters[0])
# Set the size to be 256x4x4x4
s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox,
n_gru_vox)
# Dummy 3D grid hidden representations
prev_s = InputLayer(s_shape)
t_x_s_update = FCConv3DLayer(
prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
t_x_s_reset = FCConv3DLayer(
prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
reset_gate = SigmoidLayer(t_x_s_reset)
rs = EltwiseMultiplyLayer(reset_gate, prev_s)
t_x_rs = FCConv3DLayer(rs, fc7,
(n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3),
params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3),
params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3),
params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1),
params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3),
params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3),
params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1),
params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3),
params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3),
params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3),
params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3),
params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1),
params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3),
params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3),
params=conv6b.params)
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
flat6_ = FlattenLayer(pool6_)
fc7_ = TensorProductLayer(flat6_,
n_fc_filters[0],
params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_.output, update_gate_.output
s_update, _ = theano.scan(
recurrence,
sequences=[
self.x
], # along with images, feed in the index of the current frame
outputs_info=[
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX),
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX)
])
update_all = s_update[-1]
s_all = s_update[0]
s_last = s_all[-1]
gru_s = InputLayer(s_shape, s_last)
unpool7 = Unpool3DLayer(gru_s)
conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3))
rect7a = LeakyReLU(conv7a)
conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3))
rect7 = LeakyReLU(conv7b)
res7 = AddLayer(unpool7, rect7)
unpool8 = Unpool3DLayer(res7)
conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3))
rect8a = LeakyReLU(conv8a)
conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3))
rect8 = LeakyReLU(conv8b)
res8 = AddLayer(unpool8, rect8)
# unpool9 = Unpool3DLayer(res8)
conv9a = Conv3DLayer(res8, (n_deconvfilter[3], 3, 3, 3))
rect9a = LeakyReLU(conv9a)
conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3))
rect9 = LeakyReLU(conv9b)
conv9c = Conv3DLayer(conv9a, (n_deconvfilter[3], 1, 1, 1))
res9 = AddLayer(conv9c, rect9)
conv10a = Conv3DLayer(res9, (n_deconvfilter[4], 3, 3, 3))
rect10a = LeakyReLU(conv10a)
conv10b = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3))
rect10 = LeakyReLU(conv10b)
conv10c = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3))
res10 = AddLayer(conv10c, rect10)
conv11 = Conv3DLayer(res10, (n_deconvfilter[5], 3, 3, 3))
#Debug
# con12_mat = tensor.matrix(conv12.output)
# con12_mat.tag.test_value = np.random.rand(24, 16, 2, 16, 16)
softmax_loss = SoftmaxWithLoss3D(conv11.output)
self.loss = softmax_loss.loss(self.y)
self.error = softmax_loss.error(self.y)
self.params = get_trainable_params()
self.output = softmax_loss.prediction()
self.activations = [update_all]
def recurrence(x_curr, prev_h_tensor, prev_s_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3),
params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3),
params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3),
params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1),
params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3),
params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3),
params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1),
params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3),
params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3),
params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3),
params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3),
params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1),
params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3),
params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3),
params=conv6b.params)
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
flat6_ = FlattenLayer(pool6_)
fc7_ = TensorProductLayer(flat6_,
n_fc_filters[0],
params=fc7.params)
rect7_ = LeakyReLU(fc7_)
#LSTM
# Dummy 3D grid hidden representations for previous hidden state and cell state
prev_h_ = InputLayer(h_shape, prev_h_tensor)
prev_s_ = InputLayer(h_shape, prev_s_tensor)
t_x_s_forget_ = FCConv3DLayer(
prev_h_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_forget.params)
t_x_s_input_ = FCConv3DLayer(
prev_h_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_input.params)
t_x_s_cell_ = FCConv3DLayer(
prev_h_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_cell.params)
forget_gate_ = SigmoidLayer(t_x_s_forget_)
input_gate_ = SigmoidLayer(t_x_s_input_)
tanh_t_x_s_cell_ = TanhLayer(t_x_s_cell_)
#current cell state
cell_state_ = AddLayer(
EltwiseMultiplyLayer(forget_gate_, prev_s_),
EltwiseMultiplyLayer(input_gate_, tanh_t_x_s_cell_))
#current hidden state, i.e. the output of lstm
hidden_state_ = TanhLayer(cell_state_)
return hidden_state_.output, cell_state_.output
def network_definition(self):
# (multi_views, self.batch_size, 3, self.img_h, self.img_w),
self.x = tensor5()
self.is_x_tensor4 = False
img_w = self.img_w
img_h = self.img_h
n_gru_vox = 4
# n_vox = self.n_vox
n_convfilter = [96, 128, 256, 256, 256, 256]
n_fc_filters = [1024]
n_deconvfilter = [128, 128, 128, 64, 32, 2]
input_shape = (self.batch_size, 3, img_w, img_h)
# To define weights, define the network structure first
x = InputLayer(input_shape)
conv1a = ConvLayer(x, (n_convfilter[0], 7, 7))
conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3))
pool1 = PoolLayer(conv1b)
conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3))
conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3))
conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1))
pool2 = PoolLayer(conv2c)
conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3))
conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3))
conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1))
pool3 = PoolLayer(conv3b)
conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3))
conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3))
pool4 = PoolLayer(conv4b)
conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3))
conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3))
conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1))
pool5 = PoolLayer(conv5b)
conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3))
conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3))
pool6 = PoolLayer(conv6b)
#pool6.output_shape[1] = 0
flat6 = FlattenLayer(pool6)
#flat6.output_shape[1] = 0
fc7 = TensorProductLayer(flat6, n_fc_filters[0])
# Set the size to be 256x4x4x4
s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox)
# Dummy 3D grid hidden representations
prev_s = InputLayer(s_shape)
t_x_s_update = FCConv3DLayer(prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
t_x_s_reset = FCConv3DLayer(prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
reset_gate = SigmoidLayer(t_x_s_reset)
rs = EltwiseMultiplyLayer(reset_gate, prev_s)
t_x_rs = FCConv3DLayer(rs, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3))
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params)
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
flat6_ = FlattenLayer(pool6_)
fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_.output, update_gate_.output
s_update, _ = theano.scan(recurrence,
sequences=[self.x], # along with images, feed in the index of the current frame
outputs_info=[tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX),
tensor.zeros_like(np.zeros(s_shape),
dtype=theano.config.floatX)])
# -----changes applied-----------
# with tensor.S.Session()
# pool5.output_shape[1] = 1
'''print("This is updated list function")
#theano.printing.pydotprint(s_update)
print(*s_update)
print(type(s_update[-2]))
temp = s_update[0]
#print(temp.eval())
#pp(s_update.maker.fgraph.outputs[0])
#s_update[0]
#tensor.mul(s_update[-1], 3)
print(tensor.maximum(s_update[0],s_update[1]))'''
z1 = s_update
print("This is z1", z1[0])
z1 = theano.tensor.dot(z1[0], 5)
temp = theano.tensor.dot(s_update[0], 5)
###############################################
#z2 = theano.tensor.stack([z1[0],temp[0]],2)
update_all = z1[-1]
print("This is update_all", update_all)
s_all = z1[0]
s_last = s_all[-1]
print("This is s_last shape", type(s_last))
#print("This is the value of s_last:", s_last.eval())
#print(s_last.eval(-1))
gru_s = InputLayer(s_shape, s_last)
unpool7 = Unpool3DLayer(gru_s)
conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3))
rect7a = LeakyReLU(conv7a)
conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3))
rect7 = LeakyReLU(conv7b)
res7 = AddLayer(unpool7, rect7)
unpool8 = Unpool3DLayer(res7)
conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3))
rect8a = LeakyReLU(conv8a)
conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3))
rect8 = LeakyReLU(conv8b)
res8 = AddLayer(unpool8, rect8)
unpool9 = Unpool3DLayer(res8)
conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3))
rect9a = LeakyReLU(conv9a)
conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3))
rect9 = LeakyReLU(conv9b)
conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1))
res9 = AddLayer(conv9c, rect9)
conv10a = Conv3DLayer(res9, (n_deconvfilter[4], 3, 3, 3))
rect10a = LeakyReLU(conv10a)
conv10b = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3))
rect10 = LeakyReLU(conv10b)
conv10c = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3))
res10 = AddLayer(conv10c, rect10)
conv11 = Conv3DLayer(res10, (n_deconvfilter[5], 3, 3, 3))
softmax_loss = SoftmaxWithLoss3D(conv11.output)
self.loss = softmax_loss.loss(self.y)
self.error = softmax_loss.error(self.y)
self.params = get_trainable_params()
self.output = softmax_loss.prediction()
update_all = theano.tensor.dot(update_all[0], 5)
self.activations = [update_all]