def gru3(curr_s3, prev_s3):
curr_s3 = tensor.reshape(curr_s3, s_shape3)
curr_s3_ = InputLayer(s_shape3, curr_s3)
prev_s3_ = InputLayer(s_shape3, prev_s3)
t_x_s_update3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_s_update3.params)
t_x_s_reset3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_s_reset3.params)
update3_ = SigmoidLayer(t_x_s_update3_)
comp_udpate_gate3_ = ComplementLayer(update3_)
reset_gate3_ = SigmoidLayer(t_x_s_reset3_)
rs3_ = EltwiseMultiplyLayer(reset_gate3_, prev_s3_)
t_x_rs3_ = CConv3DLayer(rs3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3),
params=t_x_rs3.params)
tanh_t_x_rs3_ = TanhLayer(t_x_rs3_)
gru_out3_ = AddLayer(
EltwiseMultiplyLayer(update3_, prev_s3_),
EltwiseMultiplyLayer(comp_udpate_gate3_, tanh_t_x_rs3_))
print("gru_out3: ", gru_out3_.output_shape)
return gru_out3_.output
def gru2(curr_s2, prev_s2):
curr_s2 = tensor.reshape(curr_s2, s_shape2)
curr_s2_ = InputLayer(s_shape2, curr_s2)
prev_s2_ = InputLayer(s_shape2, prev_s2)
t_x_s_update2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_s_update2.params)
t_x_s_reset2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_s_reset2.params)
update2_ = SigmoidLayer(t_x_s_update2_)
comp_udpate_gate2_ = ComplementLayer(update2_)
reset_gate2_ = SigmoidLayer(t_x_s_reset2_)
rs2_ = EltwiseMultiplyLayer(reset_gate2_, prev_s2_)
t_x_rs2_ = CConv3DLayer(rs2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3),
params=t_x_rs2.params)
tanh_t_x_rs2_ = TanhLayer(t_x_rs2_)
gru_out2_ = AddLayer(
EltwiseMultiplyLayer(update2_, prev_s2_),
EltwiseMultiplyLayer(comp_udpate_gate2_, tanh_t_x_rs2_))
print("gru_out2: ", gru_out2_.output_shape)
return gru_out2_.output
def gru4(curr_s4, prev_s4):
curr_s4 = tensor.reshape(curr_s4, s_shape4)
curr_s4_ = InputLayer(s_shape4, curr_s4)
prev_s4_ = InputLayer(s_shape4, prev_s4)
t_x_s_update4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_update4.params)
t_x_s_reset4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_reset4.params)
update4_ = SigmoidLayer(t_x_s_update4_)
comp_udpate_gate4_ = ComplementLayer(update4_)
reset_gate4_ = SigmoidLayer(t_x_s_reset4_)
rs4_ = EltwiseMultiplyLayer(reset_gate4_, prev_s4_)
t_x_rs4_ = CConv3DLayer(rs4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_rs4.params)
tanh_t_x_rs4_ = TanhLayer(t_x_rs4_)
gru_out4_ = AddLayer(
EltwiseMultiplyLayer(update4_, prev_s4_),
EltwiseMultiplyLayer(comp_udpate_gate4_, tanh_t_x_rs4_))
print("gru_out4: ", gru_out4_.output_shape)
return gru_out4_.output
def gru5(curr_s5, prev_s5):
curr_s5 = tensor.reshape(curr_s5, s_shape5)
curr_s5_ = InputLayer(s_shape5, curr_s5)
prev_s5_ = InputLayer(s_shape5, prev_s5)
print("curr_s5: ", curr_s5_.output_shape)
print("prev_s5: ", prev_s5_.output_shape)
t_x_s_update5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_s_update5.params)
t_x_s_reset5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_s_reset5.params)
update5_ = SigmoidLayer(t_x_s_update5_)
comp_udpate_gate5_ = ComplementLayer(update5_)
reset_gate5_ = SigmoidLayer(t_x_s_reset5_)
rs5_ = EltwiseMultiplyLayer(reset_gate5_, prev_s5_)
t_x_rs5_ = CConv3DLayer(rs5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3),
params=t_x_rs5.params)
tanh_t_x_rs5_ = TanhLayer(t_x_rs5_)
print("t_x_s_update5: ", t_x_s_update5_.output_shape)
print("t_x_s_reset5: ", t_x_s_reset5_.output_shape)
gru_out5_ = AddLayer(
EltwiseMultiplyLayer(update5_, prev_s5_),
EltwiseMultiplyLayer(comp_udpate_gate5_, tanh_t_x_rs5_))
print("gru_out5: ", gru_out5_.output_shape)
return gru_out5_.output
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1_ = ConvLayer(input_, (n_convfilter[0], 7, 7),
params=conv1.params)
pool1_ = PoolLayer(conv1_)
rect1_ = LeakyReLU(pool1_)
conv2_ = ConvLayer(rect1_, (n_convfilter[1], 3, 3),
params=conv2.params)
pool2_ = PoolLayer(conv2_)
rect2_ = LeakyReLU(pool2_)
conv3_ = ConvLayer(rect2_, (n_convfilter[2], 3, 3),
params=conv3.params)
pool3_ = PoolLayer(conv3_)
rect3_ = LeakyReLU(pool3_)
conv4_ = ConvLayer(rect3_, (n_convfilter[3], 3, 3),
params=conv4.params)
pool4_ = PoolLayer(conv4_)
rect4_ = LeakyReLU(pool4_)
conv5_ = ConvLayer(rect4_, (n_convfilter[4], 3, 3),
params=conv5.params)
pool5_ = PoolLayer(conv5_)
rect5_ = LeakyReLU(pool5_)
conv6_ = ConvLayer(rect5_, (n_convfilter[5], 3, 3),
params=conv6.params)
pool6_ = PoolLayer(conv6_)
rect6_ = LeakyReLU(pool6_)
flat6_ = FlattenLayer(rect6_)
fc7_ = TensorProductLayer(flat6_,
n_fc_filters[0],
params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_.output, update_gate_.output
def decode_recurrence_2(x_curr, prev_s_tensor, prev_in_gate_tensor):
x_curr_ = InputLayer(fc_shape, x_curr)
prev_s_2_ = InputLayer(s_shape_2, prev_s_tensor)
t_x_s_update_2_ = FCConv3DLayer(
prev_s_2_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_update_2.params)
t_x_s_reset_2_ = FCConv3DLayer(
prev_s_2_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_s_reset_2.params)
update_gate_ = SigmoidLayer(t_x_s_update_2_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_2_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_2_)
t_x_rs_2_ = FCConv3DLayer(
rs_,
x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3),
params=t_x_rs_2.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_2_)
gru_out_2_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_2_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_))
return gru_out_2_.output, update_gate_.output
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor):
# Input layer
input_ = InputLayer(input_shape, x_curr)
# GRU network same parameters as encoder
# Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv1a_ = ConvLayer(input_, (gru_filters[0], 7, 7), params=conv1a.params) # 96 x 7 x 7
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (gru_filters[0], 3, 3), params=conv1b.params) # 96 x 3 x 3
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv2a_ = ConvLayer(pool1_, (gru_filters[1], 3, 3), params=conv2a.params) # 128 x 3 x 3
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (gru_filters[1], 3, 3), params=conv2b.params) # 128 x 3 x 3
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (gru_filters[1], 1, 1), params=conv2c.params) # 128 x 1 x 1
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv3a_ = ConvLayer(pool2_, (gru_filters[2], 3, 3), params=conv3a.params) # 256 x 3 x 3
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (gru_filters[2], 3, 3), params=conv3b.params) # 256 x 3 x 3
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (gru_filters[2], 1, 1), params=conv3c.params) # 256 x 1 x 1
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
# Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv4a_ = ConvLayer(pool3_, (gru_filters[3], 3, 3), params=conv4a.params) # 256 x 3 x 3
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (gru_filters[3], 3, 3), params=conv4b.params) # 256 x 3 x 3
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv5a_ = ConvLayer(pool4_, (gru_filters[4], 3, 3), params=conv5a.params) # 256 x 3 x 3
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (gru_filters[4], 3, 3), params=conv5b.params) # 256 x 3 x 3
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (gru_filters[4], 1, 1), params=conv5c.params) # 256 x 1 x 1
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
# Residual |=> -----------------=V
# Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling
conv6a_ = ConvLayer(pool5_, (gru_filters[5], 3, 3), params=conv6a.params) # 256 x 3 x 3
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (gru_filters[5], 3, 3), params=conv6b.params) # 256 x 3 x 3
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
# Flatten Layer
flat6_ = FlattenLayer(pool6_)
# Fully connected layer
fc7_ = TensorProductLayer(flat6_, fully_connecter_filter[0], params=fc7.params)
rect7_ = LeakyReLU(fc7_)
# h(t-1)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
# FC layer convoluted with hidden states
update_layer_ = FCConv3DLayer(
prev_s_,
rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3
params=update_layer.params)
# FC layer convoluted with hidden states
reset_layer_ = FCConv3DLayer(
prev_s_,
rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3
params=reset_layer.params)
# Sigmoid( Wfx T(xt) (+) Uf * h(t-1) + bf )
update_gate_ = SigmoidLayer(update_layer_)
# 1 - u(t)
compliment_update_gate_ = ComplementLayer(update_gate_)
# Sigmoid (Wix T(xt) (+) Ui * h(t-1) + bi)
reset_gate_ = SigmoidLayer(reset_layer_)
# rt (.) h(t-1)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
# Uh * rt (.) h(t-1) + bh
hidden_layer_ = FCConv3DLayer(
rs_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), params=hidden_state_layer.params) # 128 x 3 x 3 x 3
tanh_layer = TanhLayer(hidden_layer_)
# ht = (1 - ut) (.) h(t-1) (+) tanh( Uh * rt (.) h(t-1) + bh )
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(compliment_update_gate_, tanh_layer))
return gru_out_.output, update_gate_.output
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # n_deconvfilter = [128, 128, 128, 64, 32, 2]
# Scan function cannot use compiled function.
input_ = InputLayer(input_shape, x_curr)
conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params)
rect1a_ = LeakyReLU(conv1a_)
conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params)
rect1_ = LeakyReLU(conv1b_)
pool1_ = PoolLayer(rect1_)
conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params)
rect2a_ = LeakyReLU(conv2a_)
conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params)
rect2_ = LeakyReLU(conv2b_)
conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params)
res2_ = AddLayer(conv2c_, rect2_)
pool2_ = PoolLayer(res2_)
conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params)
rect3a_ = LeakyReLU(conv3a_)
conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params)
rect3_ = LeakyReLU(conv3b_)
conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params)
res3_ = AddLayer(conv3c_, rect3_)
pool3_ = PoolLayer(res3_)
conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params)
rect4a_ = LeakyReLU(conv4a_)
conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params)
rect4_ = LeakyReLU(conv4b_)
pool4_ = PoolLayer(rect4_)
conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params)
rect5a_ = LeakyReLU(conv5a_)
conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params)
rect5_ = LeakyReLU(conv5b_)
conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params)
res5_ = AddLayer(conv5c_, rect5_)
pool5_ = PoolLayer(res5_)
conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params)
rect6a_ = LeakyReLU(conv6a_)
conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params)
rect6_ = LeakyReLU(conv6b_)
res6_ = AddLayer(pool5_, rect6_)
pool6_ = PoolLayer(res6_)
flat6_ = FlattenLayer(pool6_)
fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params)
rect7_ = LeakyReLU(fc7_)
prev_s_ = InputLayer(s_shape, prev_s_tensor)
t_x_s_update_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_update.params)
t_x_s_reset_ = FCConv3DLayer(
prev_s_,
rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3),
params=t_x_s_reset.params)
update_gate_ = SigmoidLayer(t_x_s_update_)
comp_update_gate_ = ComplementLayer(update_gate_)
reset_gate_ = SigmoidLayer(t_x_s_reset_)
rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_)
t_x_rs_ = FCConv3DLayer(
rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params)
tanh_t_x_rs_ = TanhLayer(t_x_rs_)
gru_out_ = AddLayer(
EltwiseMultiplyLayer(update_gate_, prev_s_),
EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) # (B, 128, 4, 4, 4)
return gru_out_.output, update_gate_.output