def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1.params) pool1_ = PoolLayer(conv1_) rect1_ = LeakyReLU(pool1_) conv2_ = ConvLayer(rect1_, (n_convfilter[1], 3, 3), params=conv2.params) pool2_ = PoolLayer(conv2_) rect2_ = LeakyReLU(pool2_) conv3_ = ConvLayer(rect2_, (n_convfilter[2], 3, 3), params=conv3.params) pool3_ = PoolLayer(conv3_) rect3_ = LeakyReLU(pool3_) conv4_ = ConvLayer(rect3_, (n_convfilter[3], 3, 3), params=conv4.params) pool4_ = PoolLayer(conv4_) rect4_ = LeakyReLU(pool4_) conv5_ = ConvLayer(rect4_, (n_convfilter[4], 3, 3), params=conv5.params) pool5_ = PoolLayer(conv5_) rect5_ = LeakyReLU(pool5_) conv6_ = ConvLayer(rect5_, (n_convfilter[5], 3, 3), params=conv6.params) pool6_ = PoolLayer(conv6_) rect6_ = LeakyReLU(pool6_) flat6_ = FlattenLayer(rect6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) prev_s_ = InputLayer(s_shape, prev_s_tensor) t_x_s_update_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_update.params) t_x_s_reset_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_reset.params) update_gate_ = SigmoidLayer(t_x_s_update_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) t_x_rs_ = FCConv3DLayer( rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_.output, update_gate_.output
def gru4(curr_s4, prev_s4): curr_s4 = tensor.reshape(curr_s4, s_shape4) curr_s4_ = InputLayer(s_shape4, curr_s4) prev_s4_ = InputLayer(s_shape4, prev_s4) t_x_s_update4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_s_update4.params) t_x_s_reset4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_s_reset4.params) update4_ = SigmoidLayer(t_x_s_update4_) comp_udpate_gate4_ = ComplementLayer(update4_) reset_gate4_ = SigmoidLayer(t_x_s_reset4_) rs4_ = EltwiseMultiplyLayer(reset_gate4_, prev_s4_) t_x_rs4_ = CConv3DLayer(rs4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_rs4.params) tanh_t_x_rs4_ = TanhLayer(t_x_rs4_) gru_out4_ = AddLayer( EltwiseMultiplyLayer(update4_, prev_s4_), EltwiseMultiplyLayer(comp_udpate_gate4_, tanh_t_x_rs4_)) print("gru_out4: ", gru_out4_.output_shape) return gru_out4_.output
def gru5(curr_s5, prev_s5): curr_s5 = tensor.reshape(curr_s5, s_shape5) curr_s5_ = InputLayer(s_shape5, curr_s5) prev_s5_ = InputLayer(s_shape5, prev_s5) print("curr_s5: ", curr_s5_.output_shape) print("prev_s5: ", prev_s5_.output_shape) t_x_s_update5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), params=t_x_s_update5.params) t_x_s_reset5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), params=t_x_s_reset5.params) update5_ = SigmoidLayer(t_x_s_update5_) comp_udpate_gate5_ = ComplementLayer(update5_) reset_gate5_ = SigmoidLayer(t_x_s_reset5_) rs5_ = EltwiseMultiplyLayer(reset_gate5_, prev_s5_) t_x_rs5_ = CConv3DLayer(rs5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), params=t_x_rs5.params) tanh_t_x_rs5_ = TanhLayer(t_x_rs5_) print("t_x_s_update5: ", t_x_s_update5_.output_shape) print("t_x_s_reset5: ", t_x_s_reset5_.output_shape) gru_out5_ = AddLayer( EltwiseMultiplyLayer(update5_, prev_s5_), EltwiseMultiplyLayer(comp_udpate_gate5_, tanh_t_x_rs5_)) print("gru_out5: ", gru_out5_.output_shape) return gru_out5_.output
def decode_recurrence_2(x_curr, prev_s_tensor, prev_in_gate_tensor): x_curr_ = InputLayer(fc_shape, x_curr) prev_s_2_ = InputLayer(s_shape_2, prev_s_tensor) t_x_s_update_2_ = FCConv3DLayer( prev_s_2_, x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_s_update_2.params) t_x_s_reset_2_ = FCConv3DLayer( prev_s_2_, x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_s_reset_2.params) update_gate_ = SigmoidLayer(t_x_s_update_2_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_2_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_2_) t_x_rs_2_ = FCConv3DLayer( rs_, x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_rs_2.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_2_) gru_out_2_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_2_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_2_.output, update_gate_.output
def gru3(curr_s3, prev_s3): curr_s3 = tensor.reshape(curr_s3, s_shape3) curr_s3_ = InputLayer(s_shape3, curr_s3) prev_s3_ = InputLayer(s_shape3, prev_s3) t_x_s_update3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_s_update3.params) t_x_s_reset3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_s_reset3.params) update3_ = SigmoidLayer(t_x_s_update3_) comp_udpate_gate3_ = ComplementLayer(update3_) reset_gate3_ = SigmoidLayer(t_x_s_reset3_) rs3_ = EltwiseMultiplyLayer(reset_gate3_, prev_s3_) t_x_rs3_ = CConv3DLayer(rs3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_rs3.params) tanh_t_x_rs3_ = TanhLayer(t_x_rs3_) gru_out3_ = AddLayer( EltwiseMultiplyLayer(update3_, prev_s3_), EltwiseMultiplyLayer(comp_udpate_gate3_, tanh_t_x_rs3_)) print("gru_out3: ", gru_out3_.output_shape) return gru_out3_.output
def gru2(curr_s2, prev_s2): curr_s2 = tensor.reshape(curr_s2, s_shape2) curr_s2_ = InputLayer(s_shape2, curr_s2) prev_s2_ = InputLayer(s_shape2, prev_s2) t_x_s_update2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_s_update2.params) t_x_s_reset2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_s_reset2.params) update2_ = SigmoidLayer(t_x_s_update2_) comp_udpate_gate2_ = ComplementLayer(update2_) reset_gate2_ = SigmoidLayer(t_x_s_reset2_) rs2_ = EltwiseMultiplyLayer(reset_gate2_, prev_s2_) t_x_rs2_ = CConv3DLayer(rs2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_rs2.params) tanh_t_x_rs2_ = TanhLayer(t_x_rs2_) gru_out2_ = AddLayer( EltwiseMultiplyLayer(update2_, prev_s2_), EltwiseMultiplyLayer(comp_udpate_gate2_, tanh_t_x_rs2_)) print("gru_out2: ", gru_out2_.output_shape) return gru_out2_.output
def __init__(self, input_size, output_size, hidden_layer_sizes): self.learning_rate = 0.1 self.input_layer = InputLayer(input_size) self.output_layer = OutputLayer(output_size) self.hidden_layers = [ HiddenLayer(hidden_layer_size) for hidden_layer_size in hidden_layer_sizes ] for i, hidden_layer in enumerate(self.hidden_layers): if i == 0 and i == len(self.hidden_layers) - 1: hidden_layer.initialize(self.input_layer, self.output_layer) elif i == 0: hidden_layer.initialize(self.input_layer, self.hidden_layers[i + 1]) elif i == len(self.hidden_layers) - 1: hidden_layer.initialize(self.hidden_layers[i - 1], self.output_layer) else: hidden_layer.initialize(self.hidden_layers[i - 1], self.hidden_layers[i + 1]) if (len(self.hidden_layers)): self.output_layer.initialize(self.hidden_layers[-1]) else: self.output_layer.initialize(self.input_layer)
class FeedForwardNeuralNetwork: def __init__(self, input_size, output_size, hidden_layer_sizes): self.learning_rate = 0.1 self.input_layer = InputLayer(input_size) self.output_layer = OutputLayer(output_size) self.hidden_layers = [ HiddenLayer(hidden_layer_size) for hidden_layer_size in hidden_layer_sizes ] for i, hidden_layer in enumerate(self.hidden_layers): if i == 0 and i == len(self.hidden_layers) - 1: hidden_layer.initialize(self.input_layer, self.output_layer) elif i == 0: hidden_layer.initialize(self.input_layer, self.hidden_layers[i + 1]) elif i == len(self.hidden_layers) - 1: hidden_layer.initialize(self.hidden_layers[i - 1], self.output_layer) else: hidden_layer.initialize(self.hidden_layers[i - 1], self.hidden_layers[i + 1]) if (len(self.hidden_layers)): self.output_layer.initialize(self.hidden_layers[-1]) else: self.output_layer.initialize(self.input_layer) def predict(self, input_arr): self.input_layer.set_values(input_arr) for hidden_layer in self.hidden_layers: hidden_layer.feed_forward() self.output_layer.feed_forward() return self.output_layer.values def train(self, input_arr, target_arr): self.predict(input_arr) self.output_layer.calculate_errors(target_arr) for hidden_layer in reversed(self.hidden_layers): hidden_layer.calculate_errors() self.output_layer.adjust_parameters(self.learning_rate) for hidden_layer in reversed(self.hidden_layers): hidden_layer.adjust_parameters(self.learning_rate)
def encoder(x): input_ = InputLayer(input_shape, x) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1b_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1b_, padding=(0, 0)) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2b_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2b_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3b_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3b_) pool3_ = PoolLayer(res3_, padding=(0, 0)) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4b_ = LeakyReLU(conv4b_) conv4c_ = ConvLayer(pool3_, (n_convfilter[3], 1, 1), params=conv4c.params) res4_ = AddLayer(conv4c_, rect4b_) pool4_ = PoolLayer(res4_, padding=(0, 0)) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5b_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5b_) pool5_ = PoolLayer(res5_, padding=(0, 0)) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6b_ = LeakyReLU(conv6b_) conv6c_ = ConvLayer(pool5_, (n_convfilter[5], 1, 1), params=conv6c.params) res6_ = AddLayer(conv6c_, rect6b_) flat3_ = FlattenLayer(res3_) flat4_ = FlattenLayer(res4_) flat5_ = FlattenLayer(res5_) flat6_ = FlattenLayer(res6_) print("res3: ", res3_.output_shape) print("res4: ", res4_.output_shape) print("res5: ", res5_.output_shape) print("res6: ", res6_.output_shape) # pool6_ = PoolLayer(res6_) return flat3_.output, flat4_.output, flat5_.output, flat6_.output
def encode_recurrence(x_curr): input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) # flat1_ = FlattenLayer(pool1_) # fc1_ = TensorProductLayer(flat1_, n_fc_filters[0], params=fc1.params) # out1_ = LeakyReLU(fc1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) flat2_ = FlattenLayer(pool2_) fc2_ = TensorProductLayer(flat2_, n_fc_filters[0], params=fc2.params) out2_ = LeakyReLU(fc2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) flat3_ = FlattenLayer(pool3_) fc3_ = TensorProductLayer(flat3_, n_fc_filters[0], params=fc3.params) out3_ = LeakyReLU(fc3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) flat4_ = FlattenLayer(pool4_) fc4_ = TensorProductLayer(flat4_, n_fc_filters[0], params=fc4.params) out4_ = LeakyReLU(fc4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) flat5_ = FlattenLayer(pool5_) fc5_ = TensorProductLayer(flat5_, n_fc_filters[0], params=fc5.params) out5_ = LeakyReLU(fc5_) return out5_.output, out4_.output, out3_.output, out2_.output # , out1_.output
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # n_deconvfilter = [128, 128, 128, 64, 32, 2] # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) flat6_ = FlattenLayer(pool6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) prev_s_ = InputLayer(s_shape, prev_s_tensor) t_x_s_update_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_update.params) t_x_s_reset_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_reset.params) update_gate_ = SigmoidLayer(t_x_s_update_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) t_x_rs_ = FCConv3DLayer( rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) # (B, 128, 4, 4, 4) return gru_out_.output, update_gate_.output
def network_definition(self): self.x = tensor5() self.is_x_tensor4 = False img_w = self.img_w img_h = self.img_h n_gru_vox = [32, 32, 16, 8, 4] n_convfilter = [16, 32, 64, 128, 256, 512] n_deconvfilter = [2, 2, 8, 32, 128] input_shape = (self.batch_size, 3, img_w, img_h) x = InputLayer(input_shape) conv1a = ConvLayer(x, (n_convfilter[0], 7, 7)) conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3)) pool1 = PoolLayer(conv1b, padding=(0, 0)) # H/2->64 conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3)) conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3)) conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1)) pool2 = PoolLayer(conv2c) # H/4->32 conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3)) conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3)) conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1)) pool3 = PoolLayer(conv3c, padding=(0, 0)) # H/8->16 conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3)) conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3)) conv4c = ConvLayer(pool3, (n_convfilter[3], 1, 1)) pool4 = PoolLayer(conv4c, padding=(0, 0)) # H/16->8 conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3)) conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3)) conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1)) # H/32->4 pool5 = PoolLayer(conv5c, padding=(0, 0)) conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3)) conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3)) conv6c = ConvLayer(pool5, (n_convfilter[5], 1, 1)) # H/32->4 def encoder(x): input_ = InputLayer(input_shape, x) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1b_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1b_, padding=(0, 0)) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2b_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2b_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3b_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3b_) pool3_ = PoolLayer(res3_, padding=(0, 0)) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4b_ = LeakyReLU(conv4b_) conv4c_ = ConvLayer(pool3_, (n_convfilter[3], 1, 1), params=conv4c.params) res4_ = AddLayer(conv4c_, rect4b_) pool4_ = PoolLayer(res4_, padding=(0, 0)) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5b_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5b_) pool5_ = PoolLayer(res5_, padding=(0, 0)) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6b_ = LeakyReLU(conv6b_) conv6c_ = ConvLayer(pool5_, (n_convfilter[5], 1, 1), params=conv6c.params) res6_ = AddLayer(conv6c_, rect6b_) flat3_ = FlattenLayer(res3_) flat4_ = FlattenLayer(res4_) flat5_ = FlattenLayer(res5_) flat6_ = FlattenLayer(res6_) print("res3: ", res3_.output_shape) print("res4: ", res4_.output_shape) print("res5: ", res5_.output_shape) print("res6: ", res6_.output_shape) # pool6_ = PoolLayer(res6_) return flat3_.output, flat4_.output, flat5_.output, flat6_.output # return flat6_.output # Set the shape of each resolution s_shape5 = (self.batch_size, n_gru_vox[4], n_deconvfilter[4], n_gru_vox[4], n_gru_vox[4]) s_shape4 = (self.batch_size, n_gru_vox[3], n_deconvfilter[3], n_gru_vox[3], n_gru_vox[3]) s_shape3 = (self.batch_size, n_gru_vox[2], n_deconvfilter[2], n_gru_vox[2], n_gru_vox[2]) s_shape2 = (self.batch_size, n_gru_vox[1], n_deconvfilter[1], n_gru_vox[1], n_gru_vox[1]) ## resolution 5 prev_s5 = InputLayer(s_shape5) curr_s5 = InputLayer(s_shape5) t_x_s_update5 = CConv3DLayer(prev_s5, curr_s5, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3)) t_x_s_reset5 = CConv3DLayer(prev_s5, curr_s5, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3)) reset_gate5 = SigmoidLayer(t_x_s_reset5) rs5 = EltwiseMultiplyLayer(reset_gate5, prev_s5) t_x_rs5 = CConv3DLayer(rs5, curr_s5, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3)) ## resolution 4 prev_s4 = InputLayer(s_shape4) curr_s4 = InputLayer(s_shape4) t_x_s_update4 = CConv3DLayer(prev_s4, curr_s4, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3)) t_x_s_reset4 = CConv3DLayer(prev_s4, curr_s4, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3)) reset_gate4 = SigmoidLayer(t_x_s_reset4) rs4 = EltwiseMultiplyLayer(reset_gate4, prev_s4) t_x_rs4 = CConv3DLayer(rs4, curr_s4, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3)) # resolution 3 prev_s3 = InputLayer(s_shape3) curr_s3 = InputLayer(s_shape3) t_x_s_update3 = CConv3DLayer(prev_s3, curr_s3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3)) t_x_s_reset3 = CConv3DLayer(prev_s3, curr_s3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3)) reset_gate3 = SigmoidLayer(t_x_s_reset3) rs3 = EltwiseMultiplyLayer(reset_gate3, prev_s3) t_x_rs3 = CConv3DLayer(rs3, curr_s3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3)) # resolution 4 prev_s2 = InputLayer(s_shape2) curr_s2 = InputLayer(s_shape2) t_x_s_update2 = CConv3DLayer(prev_s2, curr_s2, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3)) t_x_s_reset2 = CConv3DLayer(prev_s2, curr_s2, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3)) reset_gate2 = SigmoidLayer(t_x_s_reset2) rs2 = EltwiseMultiplyLayer(reset_gate2, prev_s2) t_x_rs2 = CConv3DLayer(rs2, curr_s2, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3)) def gru5(curr_s5, prev_s5): curr_s5 = tensor.reshape(curr_s5, s_shape5) curr_s5_ = InputLayer(s_shape5, curr_s5) prev_s5_ = InputLayer(s_shape5, prev_s5) print("curr_s5: ", curr_s5_.output_shape) print("prev_s5: ", prev_s5_.output_shape) t_x_s_update5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), params=t_x_s_update5.params) t_x_s_reset5_ = CConv3DLayer(prev_s5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), params=t_x_s_reset5.params) update5_ = SigmoidLayer(t_x_s_update5_) comp_udpate_gate5_ = ComplementLayer(update5_) reset_gate5_ = SigmoidLayer(t_x_s_reset5_) rs5_ = EltwiseMultiplyLayer(reset_gate5_, prev_s5_) t_x_rs5_ = CConv3DLayer(rs5_, curr_s5_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), params=t_x_rs5.params) tanh_t_x_rs5_ = TanhLayer(t_x_rs5_) print("t_x_s_update5: ", t_x_s_update5_.output_shape) print("t_x_s_reset5: ", t_x_s_reset5_.output_shape) gru_out5_ = AddLayer( EltwiseMultiplyLayer(update5_, prev_s5_), EltwiseMultiplyLayer(comp_udpate_gate5_, tanh_t_x_rs5_)) print("gru_out5: ", gru_out5_.output_shape) return gru_out5_.output def gru4(curr_s4, prev_s4): curr_s4 = tensor.reshape(curr_s4, s_shape4) curr_s4_ = InputLayer(s_shape4, curr_s4) prev_s4_ = InputLayer(s_shape4, prev_s4) t_x_s_update4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_s_update4.params) t_x_s_reset4_ = CConv3DLayer(prev_s4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_s_reset4.params) update4_ = SigmoidLayer(t_x_s_update4_) comp_udpate_gate4_ = ComplementLayer(update4_) reset_gate4_ = SigmoidLayer(t_x_s_reset4_) rs4_ = EltwiseMultiplyLayer(reset_gate4_, prev_s4_) t_x_rs4_ = CConv3DLayer(rs4_, curr_s4_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_rs4.params) tanh_t_x_rs4_ = TanhLayer(t_x_rs4_) gru_out4_ = AddLayer( EltwiseMultiplyLayer(update4_, prev_s4_), EltwiseMultiplyLayer(comp_udpate_gate4_, tanh_t_x_rs4_)) print("gru_out4: ", gru_out4_.output_shape) return gru_out4_.output def gru3(curr_s3, prev_s3): curr_s3 = tensor.reshape(curr_s3, s_shape3) curr_s3_ = InputLayer(s_shape3, curr_s3) prev_s3_ = InputLayer(s_shape3, prev_s3) t_x_s_update3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_s_update3.params) t_x_s_reset3_ = CConv3DLayer(prev_s3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_s_reset3.params) update3_ = SigmoidLayer(t_x_s_update3_) comp_udpate_gate3_ = ComplementLayer(update3_) reset_gate3_ = SigmoidLayer(t_x_s_reset3_) rs3_ = EltwiseMultiplyLayer(reset_gate3_, prev_s3_) t_x_rs3_ = CConv3DLayer(rs3_, curr_s3_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_rs3.params) tanh_t_x_rs3_ = TanhLayer(t_x_rs3_) gru_out3_ = AddLayer( EltwiseMultiplyLayer(update3_, prev_s3_), EltwiseMultiplyLayer(comp_udpate_gate3_, tanh_t_x_rs3_)) print("gru_out3: ", gru_out3_.output_shape) return gru_out3_.output def gru2(curr_s2, prev_s2): curr_s2 = tensor.reshape(curr_s2, s_shape2) curr_s2_ = InputLayer(s_shape2, curr_s2) prev_s2_ = InputLayer(s_shape2, prev_s2) t_x_s_update2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_s_update2.params) t_x_s_reset2_ = CConv3DLayer(prev_s2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_s_reset2.params) update2_ = SigmoidLayer(t_x_s_update2_) comp_udpate_gate2_ = ComplementLayer(update2_) reset_gate2_ = SigmoidLayer(t_x_s_reset2_) rs2_ = EltwiseMultiplyLayer(reset_gate2_, prev_s2_) t_x_rs2_ = CConv3DLayer(rs2_, curr_s2_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_rs2.params) tanh_t_x_rs2_ = TanhLayer(t_x_rs2_) gru_out2_ = AddLayer( EltwiseMultiplyLayer(update2_, prev_s2_), EltwiseMultiplyLayer(comp_udpate_gate2_, tanh_t_x_rs2_)) print("gru_out2: ", gru_out2_.output_shape) return gru_out2_.output s_encoder, _ = theano.scan(encoder, sequences=[self.x]) # print("self.x: ", self.x) out_encoder5 = s_encoder[3] out_encoder4 = s_encoder[2] out_encoder3 = s_encoder[1] out_encoder2 = s_encoder[0] s_gru5, _ = theano.scan(gru5, sequences=[out_encoder5], outputs_info=[tensor.zeros_like(np.zeros(s_shape5), dtype=theano.config.floatX)]) input_5 = InputLayer(s_shape5, s_gru5[-1]) # print("input_5: ", input_5.output_shape) pred5 = Conv3DLayer(input_5, (2, 3, 3, 3)) unpool5 = Unpool3DLayer(input_5) conv3d5 = Conv3DLayer(unpool5, (n_deconvfilter[3], 3, 3, 3)) rect3d5 = LeakyReLU(conv3d5) print("rect3d5: ", rect3d5.output_shape) print("recct3d5: ", rect3d5.output) s_gru4, _ = theano.scan(gru4, sequences=[out_encoder4], outputs_info=[rect3d5.output] ) input_4 = InputLayer(s_shape4, s_gru4[-1]) pred4 = Conv3DLayer(input_4, (2, 3, 3, 3)) unpool4 = Unpool3DLayer(input_4) conv3d4 = Conv3DLayer(unpool4, (n_deconvfilter[2], 3, 3, 3)) rect3d4 = LeakyReLU(conv3d4) print("rect3d4: ", rect3d4.output_shape) print("recct3d4: ", rect3d4.output) s_gru3, _ = theano.scan(gru3, sequences=[out_encoder3], outputs_info=[rect3d4.output]) input_3 = InputLayer(s_shape3, s_gru3[-1]) pred3 = Conv3DLayer(input_3, (2, 3, 3, 3)) unpool3 = Unpool3DLayer(input_3) conv3d3 = Conv3DLayer(unpool3, (n_deconvfilter[1], 3, 3, 3)) rect3d3 = LeakyReLU(conv3d3) print("rect3d3: ", rect3d3.output_shape) print("recct3d3: ", rect3d3.output) s_gru2, _ = theano.scan(gru2, sequences=[out_encoder2], outputs_info=[rect3d3.output]) input_2 = InputLayer(s_shape2, s_gru2[-1]) pred2 = Conv3DLayer(input_2, (2, 3, 3, 3)) labele_shape = self.y.shape label3 = self.y[:, 0:labele_shape[1]:2, :, 0:labele_shape[3]:2, 0:labele_shape[4]:2] label4 = self.y[:, 0:labele_shape[1]:4, :, 0:labele_shape[3]:4, 0:labele_shape[4]:4] label5 = self.y[:, 0:labele_shape[1]:8, :, 0:labele_shape[3]:8, 0:labele_shape[4]:8] # print("pred5: ", pred5.output_shape) # print("pred4: ", pred4.output_shape) # print("pred3: ", pred3.output_shape) print("pred2: ", pred2.output_shape) softmax_loss5 = SoftmaxWithLoss3D(pred5.output) softmax_loss4 = SoftmaxWithLoss3D(pred4.output) softmax_loss3 = SoftmaxWithLoss3D(pred3.output) softmax_loss2 = SoftmaxWithLoss3D(pred2.output) # self.loss = softmax_loss2.loss(self.y) self.loss = (softmax_loss5.loss(label5) + softmax_loss4.loss(label4) + softmax_loss3.loss(label3) + softmax_loss2.loss(self.y)) / 4. # self.loss = self.loss / self.error = softmax_loss2.error(self.y) self.output = softmax_loss2.prediction() self.params = get_trainable_params() self.activations = []
def network_definition(self): # (views, batch_size, 3, img_h, img_w) self.x = tensor5() self.is_x_tensor4 = False img_w = self.img_w img_h = self.img_h n_gru_vox = [4, 8, 16, 32] n_convfilter = [8, 16, 32, 64, 128] n_fc_filters = [256] n_deconvfilter = [128, 64, 32, 16, 2] input_shape = (self.batch_size, 3, img_w, img_h) fc_shape = (self.batch_size, n_fc_filters[0]) # To define the weights, define the net structure first x = InputLayer(input_shape) conv1a = ConvLayer(x, (n_convfilter[0], 7, 7)) conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3)) pool1 = PoolLayer(conv1b) # H/2 conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3)) conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3)) conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1)) pool2 = PoolLayer(conv2c) # H/4 conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3)) conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3)) conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1)) pool3 = PoolLayer(conv3c) # H/8 conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3)) conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3)) pool4 = PoolLayer(conv4b) # H/16 conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3)) conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3)) conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1)) # H/32 pool5 = PoolLayer(conv5b) flat5 = FlattenLayer(pool5) fc5 = TensorProductLayer(flat5, n_fc_filters[0]) flat4 = FlattenLayer(pool4) fc4 = TensorProductLayer(flat4, n_fc_filters[0]) flat3 = FlattenLayer(pool3) fc3 = TensorProductLayer(flat3, n_fc_filters[0]) flat2 = FlattenLayer(pool2) fc2 = TensorProductLayer(flat2, n_fc_filters[0]) # flat1 = FlattenLayer(pool1) # fc1 = TensorProductLayer(flat1, n_fc_filters[0]) # ==================== recurrence 5 ========================# s_shape_5 = (self.batch_size, n_gru_vox[0], n_deconvfilter[0], n_gru_vox[0], n_gru_vox[0]) # s_shape_5 = (self.batch_size, n_gru_vox[4], n_deconvfilter[4], n_gru_vox[4], n_gru_vox[4]) prev_s_5 = InputLayer(s_shape_5) t_x_s_update_5 = FCConv3DLayer( prev_s_5, fc5, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) t_x_s_reset_5 = FCConv3DLayer( prev_s_5, fc5, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) reset_gate_5 = SigmoidLayer(t_x_s_reset_5) rs_5 = EltwiseMultiplyLayer(reset_gate_5, prev_s_5) t_x_rs_5 = FCConv3DLayer( rs_5, fc5, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) # ==================== recurrence 4 ========================# s_shape_4 = (self.batch_size, n_gru_vox[1], n_deconvfilter[1], n_gru_vox[1], n_gru_vox[1]) prev_s_4 = InputLayer(s_shape_4) t_x_s_update_4 = FCConv3DLayer( prev_s_4, fc4, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3)) t_x_s_reset_4 = FCConv3DLayer( prev_s_4, fc4, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3)) reset_gate_4 = SigmoidLayer(t_x_s_reset_4) rs_4 = EltwiseMultiplyLayer(reset_gate_4, prev_s_4) t_x_rs_4 = FCConv3DLayer( rs_4, fc4, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3)) # =================== recurrence 3 =======================# s_shape_3 = (self.batch_size, n_gru_vox[2], n_deconvfilter[2], n_gru_vox[2], n_gru_vox[2]) prev_s_3 = InputLayer(s_shape_3) t_x_s_update_3 = FCConv3DLayer( prev_s_3, fc3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3)) t_x_s_reset_3 = FCConv3DLayer( prev_s_3, fc3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3)) reset_gate_3 = SigmoidLayer(t_x_s_reset_3) rs_3 = EltwiseMultiplyLayer(reset_gate_3, prev_s_3) t_x_rs_3 = FCConv3DLayer( rs_3, fc3, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3)) # ================== recurrence 2 =======================# s_shape_2 = (self.batch_size, n_gru_vox[3], n_deconvfilter[3], n_gru_vox[3], n_gru_vox[3]) prev_s_2 = InputLayer(s_shape_2) t_x_s_update_2 = FCConv3DLayer( prev_s_2, fc2, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3)) t_x_s_reset_2 = FCConv3DLayer( prev_s_2, fc2, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3)) reset_gate_2 = SigmoidLayer(t_x_s_reset_2) rs_2 = EltwiseMultiplyLayer(reset_gate_2, prev_s_2) t_x_rs_2 = FCConv3DLayer( rs_2, fc2, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3)) # # ================= recurrence 1 ========================# # s_shape_1 = (self.batch_size, n_gru_vox[4], n_deconvfilter[4], n_gru_vox[4], n_gru_vox[4]) # prev_s_1 = InputLayer(s_shape_1) # # t_x_s_update_1 = FCConv3DLayer(prev_s_1, fc1, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3)) # t_x_s_reset_1 = FCConv3DLayer(prev_s_1, fc1, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3)) # # reset_gate_1 = SigmoidLayer(t_x_s_reset_1) # rs_1 = EltwiseMultiplyLayer(reset_gate_1, prev_s_1) # t_x_rs_1 = FCConv3DLayer(rs_1, fc1, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3)) def encode_recurrence(x_curr): input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) # flat1_ = FlattenLayer(pool1_) # fc1_ = TensorProductLayer(flat1_, n_fc_filters[0], params=fc1.params) # out1_ = LeakyReLU(fc1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) flat2_ = FlattenLayer(pool2_) fc2_ = TensorProductLayer(flat2_, n_fc_filters[0], params=fc2.params) out2_ = LeakyReLU(fc2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) flat3_ = FlattenLayer(pool3_) fc3_ = TensorProductLayer(flat3_, n_fc_filters[0], params=fc3.params) out3_ = LeakyReLU(fc3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) flat4_ = FlattenLayer(pool4_) fc4_ = TensorProductLayer(flat4_, n_fc_filters[0], params=fc4.params) out4_ = LeakyReLU(fc4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) flat5_ = FlattenLayer(pool5_) fc5_ = TensorProductLayer(flat5_, n_fc_filters[0], params=fc5.params) out5_ = LeakyReLU(fc5_) return out5_.output, out4_.output, out3_.output, out2_.output # , out1_.output s_encoder, _ = theano.scan(encode_recurrence, sequences=[self.x]) out_5 = s_encoder[0] out_4 = s_encoder[1] out_3 = s_encoder[2] out_2 = s_encoder[3] # out_1 = s_encoder[4] def decode_recurrence_5(x_curr, prev_s_tensor, prev_in_gate_tensor): x_curr_ = InputLayer(fc_shape, x_curr) prev_s_5_ = InputLayer(s_shape_5, prev_s_tensor) t_x_s_update_5_ = FCConv3DLayer( prev_s_5_, x_curr_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_update_5.params) t_x_s_reset_5_ = FCConv3DLayer( prev_s_5_, x_curr_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_reset_5.params) update_gate_ = SigmoidLayer(t_x_s_update_5_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_5_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_5_) t_x_rs_5_ = FCConv3DLayer( rs_, x_curr_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs_5.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_5_) gru_out_5_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_5_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_5_.output, update_gate_.output s_update_5_, _ = theano.scan( decode_recurrence_5, sequences=[out_5], outputs_info=[ tensor.zeros_like(np.zeros(s_shape_5), dtype=theano.config.floatX), tensor.zeros_like(np.zeros(s_shape_5), dtype=theano.config.floatX) ]) update_all_5 = s_update_5_[-1] s_out_5 = update_all_5[0][-1] input_5 = InputLayer(s_shape_5, s_out_5) # Unpooling s_out_5 unpool5 = Unpool3DLayer(input_5) conv_out5 = Conv3DLayer(unpool5, (64, 3, 3, 3)) print("conv_out5", conv_out5.output_shape) def decode_recurrence_4(x_curr, prev_s_tensor, prev_in_gate_tensor): x_curr_ = InputLayer(fc_shape, x_curr) prev_s_4_ = InputLayer(s_shape_4, prev_s_tensor) t_x_s_update_4_ = FCConv3DLayer( prev_s_4_, x_curr_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_s_update_4.params) t_x_s_reset_4_ = FCConv3DLayer( prev_s_4_, x_curr_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_s_reset_4.params) print("x_curr: ", x_curr_.output_shape) print("prev_s_4_: ", prev_s_4_.output_shape) print("t_x_s_update_4_: ", t_x_s_update_4_.output_shape) print("t_x_s_reset_4_: ", t_x_s_reset_4_.output_shape) update_gate_ = SigmoidLayer(t_x_s_update_4_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_4_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_4_) t_x_rs_4_ = FCConv3DLayer( rs_, x_curr_, (n_deconvfilter[1], n_deconvfilter[1], 3, 3, 3), params=t_x_rs_4.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_4_) gru_out_4_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_4_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_4_.output, update_gate_.output s_update_4_, _ = theano.scan(decode_recurrence_4, sequences=[out_4], outputs_info=[ conv_out5.output, tensor.zeros_like( np.zeros(s_shape_4), dtype=theano.config.floatX) ]) update_all_4 = s_update_4_[-1] s_out_4 = update_all_4[0][-1] input_4 = InputLayer(s_shape_4, s_out_4) # Unpooling s_out_4 unpool4 = Unpool3DLayer(input_4) conv_out4 = Conv3DLayer(unpool4, (n_deconvfilter[2], 3, 3, 3)) print("conv_out_4: ", conv_out4.output_shape) print("conv_out_4: ", conv_out4.output) def decode_recurrence_3(x_curr, prev_s_tensor, prev_in_gate_tensor): x_curr_ = InputLayer(fc_shape, x_curr) prev_s_3_ = InputLayer(s_shape_3, prev_s_tensor) t_x_s_update_3_ = FCConv3DLayer( prev_s_3_, x_curr_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_s_update_3.params) t_x_s_reset_3_ = FCConv3DLayer( prev_s_3_, x_curr_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_s_reset_3.params) update_gate_ = SigmoidLayer(t_x_s_update_3_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_3_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_3_) t_x_rs_3_ = FCConv3DLayer( rs_, x_curr_, (n_deconvfilter[2], n_deconvfilter[2], 3, 3, 3), params=t_x_rs_3.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_3_) gru_out_3_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_3_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_3_.output, update_gate_.output s_update_3_, _ = theano.scan(decode_recurrence_3, sequences=[out_3], outputs_info=[ conv_out4.output, tensor.zeros_like( np.zeros(s_shape_3), dtype=theano.config.floatX) ]) update_all_3 = s_update_3_[-1] s_out_3 = update_all_3[0][-1] input_3 = InputLayer(s_shape_3, s_out_3) # Unpooling s_out_4 unpool3 = Unpool3DLayer(input_3) conv_out3 = Conv3DLayer(unpool3, (n_deconvfilter[3], 3, 3, 3)) print("conv_out_3: ", conv_out3.output_shape) print("conv_out_3: ", conv_out3.output) def decode_recurrence_2(x_curr, prev_s_tensor, prev_in_gate_tensor): x_curr_ = InputLayer(fc_shape, x_curr) prev_s_2_ = InputLayer(s_shape_2, prev_s_tensor) t_x_s_update_2_ = FCConv3DLayer( prev_s_2_, x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_s_update_2.params) t_x_s_reset_2_ = FCConv3DLayer( prev_s_2_, x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_s_reset_2.params) update_gate_ = SigmoidLayer(t_x_s_update_2_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_2_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_2_) t_x_rs_2_ = FCConv3DLayer( rs_, x_curr_, (n_deconvfilter[3], n_deconvfilter[3], 3, 3, 3), params=t_x_rs_2.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_2_) gru_out_2_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_2_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_2_.output, update_gate_.output s_update_2_, _ = theano.scan(decode_recurrence_2, sequences=[out_2], outputs_info=[ conv_out3.output, tensor.zeros_like( np.zeros(s_shape_2), dtype=theano.config.floatX) ]) update_all_2 = s_update_2_[-1] s_out_2 = update_all_2[0][-1] input_2 = InputLayer(s_shape_2, s_out_2) # Unpooling s_out_4 # unpool2 = Unpool3DLayer(input_2) # conv_out2 = Unpool3DLayer(unpool2, (n_deconvfilter[4], 3, 3, 3)) # def decode_recurrence_1(x_curr, prev_s_tensor, prev_in_gate_tensor): # x_curr_ = InputLayer(fc_shape, x_curr) # prev_s_1_ = InputLayer(s_shape_1, prev_s_tensor) # t_x_s_update_1_ = FCConv3DLayer(prev_s_1_, # x_curr_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), # params=t_x_s_update_1.params) # # t_x_s_reset_1_ = FCConv3DLayer(prev_s_1_, x_curr_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), # params=t_x_s_reset_1.params) # # update_gate_ = SigmoidLayer(t_x_s_update_1_) # comp_update_gate_ = ComplementLayer(update_gate_) # reset_gate_ = SigmoidLayer(t_x_s_reset_1_) # # rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_1_) # t_x_rs_1_ = FCConv3DLayer(rs_, x_curr_, (n_deconvfilter[4], n_deconvfilter[4], 3, 3, 3), # params=t_x_rs_1.params) # tanh_t_x_rs_ = TanhLayer(t_x_rs_1_) # # gru_out_1_ = AddLayer( # EltwiseMultiplyLayer(update_gate_, prev_s_1_), # EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) # # return gru_out_1_.output, update_gate_.output # # s_update_1_, _ = theano.scan(decode_recurrence_1, # sequences=[out_1], # outputs_info=[conv_out2.output, # tensor.zeros_like(np.zeros(s_shape_1), # dtype=theano.config.floatX)]) # update_all_1 = s_update_1_[-1] # s_out_1 = update_all_1[0][-1] # # s_out_1_input = InputLayer(s_shape_1, s_out_1) conv_out2 = Conv3DLayer(input_2, (n_deconvfilter[4], 3, 3, 3)) softmax_loss = SoftmaxWithLoss3D(conv_out2.output) print("conv_out_2: ", conv_out2.output_shape) print("conv_out_2: ", conv_out2.output) self.loss = softmax_loss.loss(self.y) self.error = softmax_loss.error(self.y) self.params = get_trainable_params() self.output = softmax_loss.prediction() self.activations = [ update_all_5, update_all_4, update_all_3, update_all_2 ]
def network_definition(self): # (multi_views, self.batch_size, 3, self.img_h, self.img_w), self.x = tensor5() self.is_x_tensor4 = False img_w = self.img_w img_h = self.img_h n_gru_vox = 4 n_vox = self.n_vox n_convfilter = [96, 128, 256, 256, 256, 256] n_fc_filters = [1024, 2] n_deconvfilter = [128, 128, 128, 128, 96, 2] n_maskconvfilter = [96, 2] n_conv_advfilter = [32, 128, 128, 128, 32] n_fc_advfilter = [1024, 2] input_shape = (self.batch_size, 3, img_w, img_h) rendering_shape = (self.batch_size, n_deconvfilter[3], img_h, img_w) voxel_shape = (self.batch_size, n_vox, n_vox, n_vox) # To define weights, define the network structure first x = InputLayer(input_shape) conv1a = ConvLayer(x, (n_convfilter[0], 7, 7), param_type='generator') conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3), param_type='generator') pool1 = PoolLayer(conv1b) conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3), param_type='generator') conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3), param_type='generator') conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1), param_type='generator') pool2 = PoolLayer(conv2c) conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3), param_type='generator') conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3), param_type='generator') conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1), param_type='generator') pool3 = PoolLayer(conv3b) conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3), param_type='generator') conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3), param_type='generator') pool4 = PoolLayer(conv4b) conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3), param_type='generator') conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3), param_type='generator') conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1), param_type='generator') pool5 = PoolLayer(conv5b) conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3), param_type='generator') conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3), param_type='generator') pool6 = PoolLayer(conv6b) flat6 = FlattenLayer(pool6) fc7 = TensorProductLayer(flat6, n_fc_filters[0], param_type='generator') # Set the size to be 256x4x4x4 s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox) # Dummy 3D grid hidden representations prev_s = InputLayer(s_shape) t_x_s_update = FCConv3DLayer( prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), param_type='generator') t_x_s_reset = FCConv3DLayer( prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), param_type='generator') reset_gate = SigmoidLayer(t_x_s_reset) rs = EltwiseMultiplyLayer(reset_gate, prev_s) t_x_rs = FCConv3DLayer(rs, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), param_type='generator') def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) flat6_ = FlattenLayer(pool6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) prev_s_ = InputLayer(s_shape, prev_s_tensor) t_x_s_update_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_update.params) t_x_s_reset_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_reset.params) update_gate_ = SigmoidLayer(t_x_s_update_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) t_x_rs_ = FCConv3DLayer( rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_.output, update_gate_.output s_update, r_update = theano.scan( recurrence, sequences=[ self.x[:, :, :3] ], # along with images, feed in the index of the current frame outputs_info=[ tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX), tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX) ]) update_all = s_update[-1] s_all = s_update[0] s_last = s_all[-1] gru_s = InputLayer(s_shape, s_last) unpool7 = Unpool3DLayer(gru_s) conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3), param_type='generator') rect7a = LeakyReLU(conv7a) conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3), param_type='generator') rect7 = LeakyReLU(conv7b) res7 = AddLayer(unpool7, rect7) unpool8 = Unpool3DLayer(res7) conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3), param_type='generator') rect8a = LeakyReLU(conv8a) conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3), param_type='generator') rect8 = LeakyReLU(conv8b) res8 = AddLayer(unpool8, rect8) unpool9 = Unpool3DLayer(res8) conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3), param_type='generator') rect9a = LeakyReLU(conv9a) conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3), param_type='generator') rect9 = LeakyReLU(conv9b) conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1), param_type='generator') res9 = AddLayer(conv9c, rect9) conv10a = Conv3DLayer(res9, (n_deconvfilter[3], 3, 3, 3), param_type='generator') rect10a = LeakyReLU(conv10a) conv10b = Conv3DLayer(rect10a, (n_deconvfilter[3], 3, 3, 3), param_type='generator') rect10 = LeakyReLU(conv10b) conv10c = Conv3DLayer(rect10a, (n_deconvfilter[3], 3, 3, 3), param_type='generator') res10 = AddLayer(conv10c, rect10) conv11 = Conv3DLayer(res10, (n_deconvfilter[4], 3, 3, 3), param_type='generator') conv12 = Conv3DLayer(conv11, (n_deconvfilter[5], 3, 3, 3), param_type='generator') voxel_loss = SoftmaxWithLoss3D(conv12.output) reconstruction = voxel_loss.prediction() voxel_input = InputLayer(voxel_shape, reconstruction[:, :, 1]) rend = RaytracingLayer(voxel_input, self.camera, img_w, img_h, self.pad_x, self.pad_y) assert not r_update, 'Unexpected update in the RNN.' self.mask_loss = tensor.nnet.nnet.binary_crossentropy( tensor.clip(rend.output[:, :, 0], 1e-7, 1.0 - 1e-7), tensor.gt(self.x[:, :, 3], 0.).astype(theano.config.floatX)).mean() self.voxel_loss = self.mask_loss self.discriminator_loss = None self.generator_loss = self.mask_loss self.error = voxel_loss.error(self.y) self.generator_params = get_trainable_params()['generator'] self.all_params = self.generator_params + self.discriminator_params self.load_params = self.generator_params self.output = reconstruction self.activations = [rend.output[:, :, 0]]
def network_definition(self): # (multi_views, self.batch_size, 3, self.img_h, self.img_w), self.x = tensor5() self.is_x_tensor4 = False img_w = self.img_w img_h = self.img_h n_gru_vox = 4 # n_vox = self.n_vox n_convfilter = [96, 128, 256, 256, 256, 256] n_fc_filters = [1024] n_deconvfilter = [128, 128, 128, 64, 32, 2] input_shape = (self.batch_size, 3, img_w, img_h) # To define weights, define the network structure first x = InputLayer(input_shape) conv1 = ConvLayer(x, (n_convfilter[0], 7, 7)) pool1 = PoolLayer(conv1) conv2 = ConvLayer(pool1, (n_convfilter[1], 3, 3)) pool2 = PoolLayer(conv2) conv3 = ConvLayer(pool2, (n_convfilter[2], 3, 3)) pool3 = PoolLayer(conv3) conv4 = ConvLayer(pool3, (n_convfilter[3], 3, 3)) pool4 = PoolLayer(conv4) conv5 = ConvLayer(pool4, (n_convfilter[4], 3, 3)) pool5 = PoolLayer(conv5) conv6 = ConvLayer(pool5, (n_convfilter[5], 3, 3)) pool6 = PoolLayer(conv6) flat6 = FlattenLayer(pool6) fc7 = TensorProductLayer(flat6, n_fc_filters[0]) # Set the size to be 256x4x4x4 s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox) # Dummy 3D grid hidden representations prev_s = InputLayer(s_shape) t_x_s_update = FCConv3DLayer( prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) t_x_s_reset = FCConv3DLayer( prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) reset_gate = SigmoidLayer(t_x_s_reset) rs = EltwiseMultiplyLayer(reset_gate, prev_s) t_x_rs = FCConv3DLayer(rs, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1.params) pool1_ = PoolLayer(conv1_) rect1_ = LeakyReLU(pool1_) conv2_ = ConvLayer(rect1_, (n_convfilter[1], 3, 3), params=conv2.params) pool2_ = PoolLayer(conv2_) rect2_ = LeakyReLU(pool2_) conv3_ = ConvLayer(rect2_, (n_convfilter[2], 3, 3), params=conv3.params) pool3_ = PoolLayer(conv3_) rect3_ = LeakyReLU(pool3_) conv4_ = ConvLayer(rect3_, (n_convfilter[3], 3, 3), params=conv4.params) pool4_ = PoolLayer(conv4_) rect4_ = LeakyReLU(pool4_) conv5_ = ConvLayer(rect4_, (n_convfilter[4], 3, 3), params=conv5.params) pool5_ = PoolLayer(conv5_) rect5_ = LeakyReLU(pool5_) conv6_ = ConvLayer(rect5_, (n_convfilter[5], 3, 3), params=conv6.params) pool6_ = PoolLayer(conv6_) rect6_ = LeakyReLU(pool6_) flat6_ = FlattenLayer(rect6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) prev_s_ = InputLayer(s_shape, prev_s_tensor) t_x_s_update_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_update.params) t_x_s_reset_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_reset.params) update_gate_ = SigmoidLayer(t_x_s_update_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) t_x_rs_ = FCConv3DLayer( rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_.output, update_gate_.output s_update, _ = theano.scan( recurrence, sequences=[ self.x ], # along with images, feed in the index of the current frame outputs_info=[ tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX), tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX) ]) update_all = s_update[-1] s_all = s_update[0] s_last = s_all[-1] gru_s = InputLayer(s_shape, s_last) unpool7 = Unpool3DLayer(gru_s) conv7 = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3)) rect7 = LeakyReLU(conv7) unpool8 = Unpool3DLayer(rect7) conv8 = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3)) rect8 = LeakyReLU(conv8) unpool9 = Unpool3DLayer(rect8) conv9 = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3)) rect9 = LeakyReLU(conv9) # unpool10 = Unpool3DLayer(rect9) conv10 = Conv3DLayer(rect9, (n_deconvfilter[4], 3, 3, 3)) rect10 = LeakyReLU(conv10) conv11 = Conv3DLayer(rect10, (n_deconvfilter[5], 3, 3, 3)) softmax_loss = SoftmaxWithLoss3D(conv11.output) self.loss = softmax_loss.loss(self.y) self.error = softmax_loss.error(self.y) self.params = get_trainable_params() self.output = softmax_loss.prediction() self.activations = [update_all]
def network_definition(self): # (multi_views, self.batch_size, 3, self.img_h, self.img_w), self.x = tensor5() self.is_x_tensor4 = False img_w = self.img_w img_h = self.img_h n_gru_vox = 4 n_vox = self.n_vox n_convfilter = [96, 128, 256, 256, 256, 256] n_fc_filters = [1024, 2] n_deconvfilter = [128, 128, 128, 128, 96, 2] n_conv_advfilter = [32, 128, 128, 128, 32] n_fc_advfilter = [1024, 2] input_shape = (self.batch_size, 3, img_w, img_h) voxel_shape = (self.batch_size, n_vox, n_vox, n_vox) # To define weights, define the network structure first x = InputLayer(input_shape) conv1a = ConvLayer(x, (n_convfilter[0], 7, 7), param_type='generator') conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3), param_type='generator') pool1 = PoolLayer(conv1b) conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3), param_type='generator') conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3), param_type='generator') conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1), param_type='generator') pool2 = PoolLayer(conv2c) conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3), param_type='generator') conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3), param_type='generator') conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1), param_type='generator') pool3 = PoolLayer(conv3b) conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3), param_type='generator') conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3), param_type='generator') pool4 = PoolLayer(conv4b) conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3), param_type='generator') conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3), param_type='generator') conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1), param_type='generator') pool5 = PoolLayer(conv5b) conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3), param_type='generator') conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3), param_type='generator') pool6 = PoolLayer(conv6b) flat6 = FlattenLayer(pool6) fc7 = TensorProductLayer(flat6, n_fc_filters[0], param_type='generator') # Set the size to be 256x4x4x4 s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox) # Dummy 3D grid hidden representations prev_s = InputLayer(s_shape) t_x_s_update = FCConv3DLayer( prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), param_type='generator') t_x_s_reset = FCConv3DLayer( prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), param_type='generator') reset_gate = SigmoidLayer(t_x_s_reset) rs = EltwiseMultiplyLayer(reset_gate, prev_s) t_x_rs = FCConv3DLayer(rs, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), param_type='generator') def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) flat6_ = FlattenLayer(pool6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) prev_s_ = InputLayer(s_shape, prev_s_tensor) t_x_s_update_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_update.params) t_x_s_reset_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_reset.params) update_gate_ = SigmoidLayer(t_x_s_update_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) t_x_rs_ = FCConv3DLayer( rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_.output, update_gate_.output s_update, r_update = theano.scan( recurrence, sequences=[ self.x[:, :, :3] ], # along with images, feed in the index of the current frame outputs_info=[ tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX), tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX) ]) update_all = s_update[-1] s_all = s_update[0] s_last = s_all[-1] gru_s = InputLayer(s_shape, s_last) unpool7 = Unpool3DLayer(gru_s) conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3), param_type='generator') rect7a = LeakyReLU(conv7a) conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3), param_type='generator') rect7 = LeakyReLU(conv7b) res7 = AddLayer(unpool7, rect7) unpool8 = Unpool3DLayer(res7) conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3), param_type='generator') rect8a = LeakyReLU(conv8a) conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3), param_type='generator') rect8 = LeakyReLU(conv8b) res8 = AddLayer(unpool8, rect8) unpool9 = Unpool3DLayer(res8) conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3), param_type='generator') rect9a = LeakyReLU(conv9a) conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3), param_type='generator') rect9 = LeakyReLU(conv9b) conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1), param_type='generator') res9 = AddLayer(conv9c, rect9) conv10a = Conv3DLayer(res9, (n_deconvfilter[3], 3, 3, 3), param_type='generator') rect10a = LeakyReLU(conv10a) conv10b = Conv3DLayer(rect10a, (n_deconvfilter[3], 3, 3, 3), param_type='generator') rect10 = LeakyReLU(conv10b) conv10c = Conv3DLayer(rect10a, (n_deconvfilter[3], 3, 3, 3), param_type='generator') res10 = AddLayer(conv10c, rect10) conv11 = Conv3DLayer(res10, (n_deconvfilter[4], 3, 3, 3), param_type='generator') conv12 = Conv3DLayer(conv11, (n_deconvfilter[5], 3, 3, 3), param_type='generator') voxel_loss = SoftmaxWithLoss3D(conv12.output) reconstruction = voxel_loss.prediction() voxel_input = InputLayer(voxel_shape, reconstruction[:, :, 1]) rend = RaytracingLayer(voxel_input, self.camera, img_w, img_h, self.pad_x, self.pad_y) # Discriminator network starts here. disc_input = InputLayer(voxel_shape) disc_padded = DimShuffleLayer(disc_input, (0, 1, 'x', 2, 3)) conv15 = Conv3DLayer(disc_padded, (n_conv_advfilter[0], 3, 3, 3), param_type='discriminator') conv16 = Conv3DLayer(conv15, (n_conv_advfilter[0], 3, 3, 3), param_type='discriminator') pool16 = Pool3DLayer(conv16) # b x 16 x c x 16 x 16 conv17 = Conv3DLayer(pool16, (n_conv_advfilter[1], 3, 3, 3), param_type='discriminator') conv18 = Conv3DLayer(conv17, (n_conv_advfilter[1], 3, 3, 3), param_type='discriminator') pool18 = Pool3DLayer(conv18) # b x 8 x c x 8 x 8 conv19 = Conv3DLayer(pool18, (n_conv_advfilter[2], 3, 3, 3), param_type='discriminator') conv20 = Conv3DLayer(conv19, (n_conv_advfilter[2], 3, 3, 3), param_type='discriminator') pool20 = Pool3DLayer(conv20) # b x 4 x c x 4 x 4 conv21 = Conv3DLayer(pool20, (n_conv_advfilter[3], 3, 3, 3), param_type='discriminator') conv22 = Conv3DLayer(conv21, (n_conv_advfilter[3], 3, 3, 3), param_type='discriminator') pool22 = Pool3DLayer(conv22) # b x 2 x c x 2 x 2 conv23 = Conv3DLayer(pool22, (n_conv_advfilter[4], 3, 3, 3), param_type='discriminator') conv24 = Conv3DLayer(conv23, (n_conv_advfilter[4], 1, 1, 1), param_type='discriminator') flat24 = FlattenLayer(conv24) fc24 = TensorProductLayer(flat24, n_fc_advfilter[1], param_type='discriminator') def get_discriminator(data_centered, use_dropout): conv15_ = Conv3DLayer(data_centered, (n_conv_advfilter[0], 3, 3, 3), params=conv15.params) rect15_ = LeakyReLU(conv15_) conv16_ = Conv3DLayer(rect15_, (n_conv_advfilter[0], 3, 3, 3), params=conv16.params) rect16_ = LeakyReLU(conv16_) pool16_ = Pool3DLayer(rect16_) # b x 16 x c x 16 x 16 conv17_ = Conv3DLayer(pool16_, (n_conv_advfilter[1], 3, 3, 3), params=conv17.params) rect17_ = LeakyReLU(conv17_) conv18_ = Conv3DLayer(rect17_, (n_conv_advfilter[1], 3, 3, 3), params=conv18.params) rect18_ = LeakyReLU(conv18_) pool18_ = Pool3DLayer(rect18_) # b x 8 x c x 8 x 8 conv19_ = Conv3DLayer(pool18_, (n_conv_advfilter[2], 3, 3, 3), params=conv19.params) rect19_ = LeakyReLU(conv19_) conv20_ = Conv3DLayer(rect19_, (n_conv_advfilter[2], 3, 3, 3), params=conv20.params) rect20_ = LeakyReLU(conv20_) pool20_ = Pool3DLayer(rect20_) # b x 4 x c x 4 x 4 conv21_ = Conv3DLayer(pool20_, (n_conv_advfilter[3], 3, 3, 3), params=conv21.params) rect21_ = LeakyReLU(conv21_) conv22_ = Conv3DLayer(rect21_, (n_conv_advfilter[3], 3, 3, 3), params=conv22.params) rect22_ = LeakyReLU(conv22_) pool22_ = Pool3DLayer(rect22_) # b x 2 x c x 2 x 2 conv23_ = Conv3DLayer(pool22_, (n_conv_advfilter[4], 3, 3, 3), params=conv23.params) rect23_ = LeakyReLU(conv23_) conv24_ = Conv3DLayer(rect23_, (n_conv_advfilter[4], 1, 1, 1), params=conv24.params) flat24_ = FlattenLayer(conv24_) fc24_ = TensorProductLayer(flat24_, n_fc_advfilter[1], params=fc24.params) return SoftmaxWithLoss3D(fc24_.output, axis=1) voxel_padded = DimShuffleLayer(voxel_input, (0, 1, 'x', 2, 3)) if cfg.TRAIN.STABILIZER == 'diffstep': voxel_stabilized = DifferentiableStepLayer( voxel_padded, backprop=cfg.TRAIN.DIFF_BACKPROP) elif cfg.TRAIN.STABILIZER == 'noise': voxel_stabilized = InstanceNoiseLayer(voxel_padded, std=self.noise * cfg.TRAIN.NOISE_MAXSTD) elif cfg.TRAIN.STABILIZER == 'ignore': voxel_stabilized = voxel_padded else: raise NotImplemented voxel_centered = SubtractLayer(voxel_stabilized, 0.5) gt_input = InputLayer(voxel_shape, self.y[:, :, 1]) gt_padded = DimShuffleLayer(gt_input, (0, 1, 'x', 2, 3)) if cfg.TRAIN.STABILIZER == 'diffstep': gt_stabilized = gt_padded elif cfg.TRAIN.STABILIZER == 'noise': gt_stabilized = InstanceNoiseLayer(gt_padded, std=self.noise * cfg.TRAIN.NOISE_MAXSTD) elif cfg.TRAIN.STABILIZER == 'ignore': gt_stabilized = gt_padded else: raise NotImplemented gt_centered = SubtractLayer(gt_stabilized, 0.5) # Discriminator 1: takes fake voxel as input. discriminator_fake_loss = get_discriminator(voxel_centered, True) # Discriminator 2: takes real voxel as input. discriminator_real_loss = get_discriminator(gt_centered, True) # Discriminator 3: takes generated voxel as input, doesn't use dropout. discriminator_fake_test = get_discriminator(voxel_centered, False) # Discriminator 4: takes real voxel as input, doesn't use dropout. discriminator_real_test = get_discriminator(gt_centered, False) assert not r_update, 'Unexpected update in the RNN.' label_shape = np.zeros((self.batch_size, 1)) fake_label = tensor.zeros_like(label_shape, dtype=theano.config.floatX) real_label = tensor.ones_like(label_shape, dtype=theano.config.floatX) all_fake = tensor.concatenate((real_label, fake_label), axis=1) all_real = tensor.concatenate((fake_label, real_label), axis=1) self.voxel_loss = discriminator_fake_test.loss(all_real) self.mask_loss = tensor.nnet.nnet.binary_crossentropy( tensor.clip(rend.output[:, :, 0], 1e-7, 1.0 - 1e-7), tensor.gt(self.x[:, :, 3], 0.).astype(theano.config.floatX)).mean() self.discriminator_loss = (discriminator_fake_loss.loss(all_fake) + discriminator_real_loss.loss(all_real)) / 2. self.generator_loss = self.voxel_loss + self.mask_loss * 100 self.error = voxel_loss.error(self.y) self.error_F = discriminator_fake_test.error(all_fake) self.error_R = discriminator_real_test.error(all_real) self.generator_params = get_trainable_params()['generator'] self.discriminator_params = get_trainable_params()['discriminator'] self.all_params = self.generator_params + self.discriminator_params self.load_params = self.all_params self.output = reconstruction self.activations = [rend.output[:, :, 0]]
def network_definition(self): # (multi_views, self.batch_size, 3, self.img_h, self.img_w), self.x = tensor5() self.is_x_tensor4 = False img_w = self.img_w img_h = self.img_h n_gru_vox = 4 # n_vox = self.n_vox n_convfilter = [96, 128, 256, 256, 256, 256] n_fc_filters = [1024] n_deconvfilter = [128, 128, 128, 64, 32, 2] input_shape = (self.batch_size, 3, img_w, img_h) # To define weights, define the network structure first x = InputLayer(input_shape) conv1a = ConvLayer(x, (n_convfilter[0], 7, 7)) conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3)) pool1 = PoolLayer(conv1b) conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3)) conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3)) conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1)) pool2 = PoolLayer(conv2c) conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3)) conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3)) conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1)) pool3 = PoolLayer(conv3b) conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3)) conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3)) pool4 = PoolLayer(conv4b) conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3)) conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3)) conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1)) pool5 = PoolLayer(conv5b) conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3)) conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3)) pool6 = PoolLayer(conv6b) flat6 = FlattenLayer(pool6) fc7 = TensorProductLayer(flat6, n_fc_filters[0]) #LSTM # Set the size to be 256x4x4x4 h_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox) # Dummy 3D grid hidden representations prev_h = InputLayer(h_shape) t_x_s_forget = FCConv3DLayer( prev_h, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) t_x_s_input = FCConv3DLayer( prev_h, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) t_x_s_cell = FCConv3DLayer( prev_h, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) #initialize hidden state and cell state with 0 if self.hidden_last is None: self.hidden_last = theano.shared( np.zeros(h_shape, dtype=theano.config.floatX)) if self.cell_last is None: self.cell_last = theano.shared( np.zeros(h_shape, dtype=theano.config.floatX)) def recurrence(x_curr, prev_h_tensor, prev_s_tensor): # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) flat6_ = FlattenLayer(pool6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) #LSTM # Dummy 3D grid hidden representations for previous hidden state and cell state prev_h_ = InputLayer(h_shape, prev_h_tensor) prev_s_ = InputLayer(h_shape, prev_s_tensor) t_x_s_forget_ = FCConv3DLayer( prev_h_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_forget.params) t_x_s_input_ = FCConv3DLayer( prev_h_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_input.params) t_x_s_cell_ = FCConv3DLayer( prev_h_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_cell.params) forget_gate_ = SigmoidLayer(t_x_s_forget_) input_gate_ = SigmoidLayer(t_x_s_input_) tanh_t_x_s_cell_ = TanhLayer(t_x_s_cell_) #current cell state cell_state_ = AddLayer( EltwiseMultiplyLayer(forget_gate_, prev_s_), EltwiseMultiplyLayer(input_gate_, tanh_t_x_s_cell_)) #current hidden state, i.e. the output of lstm hidden_state_ = TanhLayer(cell_state_) return hidden_state_.output, cell_state_.output s_update, _ = theano.scan( recurrence, sequences=[ self.x ], # along with images, feed in the index of the current frame outputs_info=[ self.hidden_last.get_value(), self.cell_last.get_value() ]) #s_update means updates of hidden states and cell states cell_all = s_update[-1] h_all = s_update[0] h_last = h_all[-1] lstm_s = InputLayer(h_shape, h_last) unpool7 = Unpool3DLayer(lstm_s) conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3)) rect7a = LeakyReLU(conv7a) conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3)) rect7 = LeakyReLU(conv7b) res7 = AddLayer(unpool7, rect7) unpool8 = Unpool3DLayer(res7) conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3)) rect8a = LeakyReLU(conv8a) conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3)) rect8 = LeakyReLU(conv8b) res8 = AddLayer(unpool8, rect8) unpool9 = Unpool3DLayer(res8) conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3)) rect9a = LeakyReLU(conv9a) conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3)) rect9 = LeakyReLU(conv9b) conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1)) res9 = AddLayer(conv9c, rect9) conv10a = Conv3DLayer(res9, (n_deconvfilter[4], 3, 3, 3)) rect10a = LeakyReLU(conv10a) conv10b = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3)) rect10 = LeakyReLU(conv10b) conv10c = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3)) res10 = AddLayer(conv10c, rect10) conv11 = Conv3DLayer(res10, (n_deconvfilter[5], 3, 3, 3)) softmax_loss = SoftmaxWithLoss3D(conv11.output) self.loss = softmax_loss.loss(self.y) self.error = softmax_loss.error(self.y) self.params = get_trainable_params() self.output = softmax_loss.prediction() self.activations = [cell_all] self.new_hidden_last = h_last self.new_cell_last = cell_all[-1]
def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # Input layer input_ = InputLayer(input_shape, x_curr) # GRU network same parameters as encoder # Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling conv1a_ = ConvLayer(input_, (gru_filters[0], 7, 7), params=conv1a.params) # 96 x 7 x 7 rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (gru_filters[0], 3, 3), params=conv1b.params) # 96 x 3 x 3 rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) # Residual |=> -----------------=V # Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling conv2a_ = ConvLayer(pool1_, (gru_filters[1], 3, 3), params=conv2a.params) # 128 x 3 x 3 rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (gru_filters[1], 3, 3), params=conv2b.params) # 128 x 3 x 3 rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (gru_filters[1], 1, 1), params=conv2c.params) # 128 x 1 x 1 res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) # Residual |=> -----------------=V # Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling conv3a_ = ConvLayer(pool2_, (gru_filters[2], 3, 3), params=conv3a.params) # 256 x 3 x 3 rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (gru_filters[2], 3, 3), params=conv3b.params) # 256 x 3 x 3 rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (gru_filters[2], 1, 1), params=conv3c.params) # 256 x 1 x 1 res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) # Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling conv4a_ = ConvLayer(pool3_, (gru_filters[3], 3, 3), params=conv4a.params) # 256 x 3 x 3 rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (gru_filters[3], 3, 3), params=conv4b.params) # 256 x 3 x 3 rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) # Residual |=> -----------------=V # Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling conv5a_ = ConvLayer(pool4_, (gru_filters[4], 3, 3), params=conv5a.params) # 256 x 3 x 3 rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (gru_filters[4], 3, 3), params=conv5b.params) # 256 x 3 x 3 rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (gru_filters[4], 1, 1), params=conv5c.params) # 256 x 1 x 1 res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) # Residual |=> -----------------=V # Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling conv6a_ = ConvLayer(pool5_, (gru_filters[5], 3, 3), params=conv6a.params) # 256 x 3 x 3 rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (gru_filters[5], 3, 3), params=conv6b.params) # 256 x 3 x 3 rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) # Flatten Layer flat6_ = FlattenLayer(pool6_) # Fully connected layer fc7_ = TensorProductLayer(flat6_, fully_connecter_filter[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) # h(t-1) prev_s_ = InputLayer(s_shape, prev_s_tensor) # FC layer convoluted with hidden states update_layer_ = FCConv3DLayer( prev_s_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3 params=update_layer.params) # FC layer convoluted with hidden states reset_layer_ = FCConv3DLayer( prev_s_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3 params=reset_layer.params) # Sigmoid( Wfx T(xt) (+) Uf * h(t-1) + bf ) update_gate_ = SigmoidLayer(update_layer_) # 1 - u(t) compliment_update_gate_ = ComplementLayer(update_gate_) # Sigmoid (Wix T(xt) (+) Ui * h(t-1) + bi) reset_gate_ = SigmoidLayer(reset_layer_) # rt (.) h(t-1) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) # Uh * rt (.) h(t-1) + bh hidden_layer_ = FCConv3DLayer( rs_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), params=hidden_state_layer.params) # 128 x 3 x 3 x 3 tanh_layer = TanhLayer(hidden_layer_) # ht = (1 - ut) (.) h(t-1) (+) tanh( Uh * rt (.) h(t-1) + bh ) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(compliment_update_gate_, tanh_layer)) return gru_out_.output, update_gate_.output
def network_definition(self): # Depth of the convolutional layers. VGG Style cnn_filters = [96, 128, 256, 256, 256, 256] # One fully connected layer for a 1024 feature vector fully_connecter_filter = [1024] # Shape of input layers. Used by encoder and GRU input_shape = (self.batch_size, 3, self.img_width, self.img_height) ######### Encoder ########## # Input Layer x = InputLayer(input_shape) ## First set of convolutional layers ## conv1a = ConvLayer(x, (cnn_filters[0], 7, 7)) # 96 x 7 x 7 conv1b = ConvLayer(conv1a, (cnn_filters[0], 3, 3)) # 96 x 3 x 3 pool1 = PoolLayer(conv1b) # Max Pooling ## Second set of convolutional layers ## conv2a = ConvLayer(pool1, (cnn_filters[1], 3, 3)) # 128 x 3 x 3 conv2b = ConvLayer(conv2a, (cnn_filters[1], 3, 3)) # 128 x 3 x 3 conv2c = ConvLayer(conv2b, (cnn_filters[1], 1, 1)) # 128 x 1 x 1 pool2 = PoolLayer(conv2c) # Max Pooling ## Third set of convolutional layers ## conv3a = ConvLayer(pool2, (cnn_filters[2], 3, 3)) # 256 x 3 x 3 conv3b = ConvLayer(conv3a, (cnn_filters[2], 3, 3)) # 256 x 3 x 3 conv3c = ConvLayer(pool2, (cnn_filters[2], 1, 1)) # 256 x 1 x 1 pool3 = PoolLayer(conv3b) # Max Pooling ## Fourth set of convolutional layers ## conv4a = ConvLayer(pool3, (cnn_filters[3], 3, 3)) # 256 x 3 x 3 conv4b = ConvLayer(conv4a, (cnn_filters[3], 3, 3)) # 256 x 3 x 3 pool4 = PoolLayer(conv4b) # Max Pooling ## Fifth set of convolutional layers ## conv5a = ConvLayer(pool4, (cnn_filters[4], 3, 3)) # 256 x 3 x 3 conv5b = ConvLayer(conv5a, (cnn_filters[4], 3, 3)) # 256 x 3 x 3 conv5c = ConvLayer(pool4, (cnn_filters[4], 1, 1)) # 256 x 1 x 1 pool5 = PoolLayer(conv5b) # Max pooling ## Sixth set of convolutional layers ## conv6a = ConvLayer(pool5, (cnn_filters[5], 3, 3)) # 256 x 3 x 3 conv6b = ConvLayer(conv6a, (cnn_filters[5], 3, 3)) # 256 x 3 x 3 pool6 = PoolLayer(conv6b) # Flatten layer flat6 = FlattenLayer(pool6) # Fully Connected layer fc7 = TensorProductLayer(flat6, 1024) # 1024 feature vector ########## End Encoder ############ ########## Gated Recurrent Unit ############ # Filter size of layers within the unit gru_filters = [96, 128, 256, 256, 256, 256] # The 3D Convolutional LSTM has a grid structure of 4 x 4 x 4. 128 for first layer of decoder s_shape = (self.batch_size, self.n_gru_vox, gru_filters[1], self.n_gru_vox, self.n_gru_vox) # Initialize the first previous state to nothing prev_s = InputLayer(s_shape) # h(t-1) # 3 x 3 x 3 Convolution of hidden states of self and neighbors # Wfx T(xt) (+) Uf * h(t-1) + bf update_layer = FCConv3DLayer(prev_s, fc7, (gru_filters[1], gru_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3 # Wix T(xt) (+) Ui * h(t-1) + bi reset_layer = FCConv3DLayer(prev_s, fc7, (gru_filters[1], gru_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3 # Sigmoid (Wix T(xt) (+) Ui * h(t-1) + bi) reset_gate = SigmoidLayer(reset_layer) # rt (.) h(t-1) rs = EltwiseMultiplyLayer(reset_gate, prev_s) # Used for h(t) # Wh T(xt) (+) Uh * (rt (.) h(t-1) + bh hidden_state_layer = FCConv3DLayer(rs, fc7, (gru_filters[1], gru_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3 # Recurrence unit def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # Input layer input_ = InputLayer(input_shape, x_curr) # GRU network same parameters as encoder # Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling conv1a_ = ConvLayer(input_, (gru_filters[0], 7, 7), params=conv1a.params) # 96 x 7 x 7 rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (gru_filters[0], 3, 3), params=conv1b.params) # 96 x 3 x 3 rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) # Residual |=> -----------------=V # Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling conv2a_ = ConvLayer(pool1_, (gru_filters[1], 3, 3), params=conv2a.params) # 128 x 3 x 3 rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (gru_filters[1], 3, 3), params=conv2b.params) # 128 x 3 x 3 rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (gru_filters[1], 1, 1), params=conv2c.params) # 128 x 1 x 1 res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) # Residual |=> -----------------=V # Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling conv3a_ = ConvLayer(pool2_, (gru_filters[2], 3, 3), params=conv3a.params) # 256 x 3 x 3 rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (gru_filters[2], 3, 3), params=conv3b.params) # 256 x 3 x 3 rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (gru_filters[2], 1, 1), params=conv3c.params) # 256 x 1 x 1 res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) # Conv -> leakyReLU -> Conv -> LeakyReLU -> MaxPooling conv4a_ = ConvLayer(pool3_, (gru_filters[3], 3, 3), params=conv4a.params) # 256 x 3 x 3 rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (gru_filters[3], 3, 3), params=conv4b.params) # 256 x 3 x 3 rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) # Residual |=> -----------------=V # Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling conv5a_ = ConvLayer(pool4_, (gru_filters[4], 3, 3), params=conv5a.params) # 256 x 3 x 3 rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (gru_filters[4], 3, 3), params=conv5b.params) # 256 x 3 x 3 rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (gru_filters[4], 1, 1), params=conv5c.params) # 256 x 1 x 1 res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) # Residual |=> -----------------=V # Conv -> leakyReLU -> Conv -> LeakyReLU -> Conv -> LeakyReLU -> MaxPooling conv6a_ = ConvLayer(pool5_, (gru_filters[5], 3, 3), params=conv6a.params) # 256 x 3 x 3 rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (gru_filters[5], 3, 3), params=conv6b.params) # 256 x 3 x 3 rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) # Flatten Layer flat6_ = FlattenLayer(pool6_) # Fully connected layer fc7_ = TensorProductLayer(flat6_, fully_connecter_filter[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) # h(t-1) prev_s_ = InputLayer(s_shape, prev_s_tensor) # FC layer convoluted with hidden states update_layer_ = FCConv3DLayer( prev_s_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3 params=update_layer.params) # FC layer convoluted with hidden states reset_layer_ = FCConv3DLayer( prev_s_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), # 128 x 3 x 3 x 3 params=reset_layer.params) # Sigmoid( Wfx T(xt) (+) Uf * h(t-1) + bf ) update_gate_ = SigmoidLayer(update_layer_) # 1 - u(t) compliment_update_gate_ = ComplementLayer(update_gate_) # Sigmoid (Wix T(xt) (+) Ui * h(t-1) + bi) reset_gate_ = SigmoidLayer(reset_layer_) # rt (.) h(t-1) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) # Uh * rt (.) h(t-1) + bh hidden_layer_ = FCConv3DLayer( rs_, rect7_, (gru_filters[1], gru_filters[1], 3, 3, 3), params=hidden_state_layer.params) # 128 x 3 x 3 x 3 tanh_layer = TanhLayer(hidden_layer_) # ht = (1 - ut) (.) h(t-1) (+) tanh( Uh * rt (.) h(t-1) + bh ) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(compliment_update_gate_, tanh_layer)) return gru_out_.output, update_gate_.output s_update, _ = theano.scan(recurrence, sequences=[self.x], # along with images, feed in the index of the current frame outputs_info=[tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX), tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX)]) # Update of all units update_all = s_update[-1] s_all = s_update[0] # Last hidden states. last timestep s_last = s_all[-1] ########## End GRU ########## ########## Decoder ########## # Depth of deconvolutional layers dcnn_filters = [128, 128, 128, 64, 32, 2] # Input Layer gru_s = InputLayer(s_shape, s_last) # Residual |=> ----------------------------------------------=V # Unpooling -> deconvolution -> LeakyReLU -> DeConv -> LeakyReLU -> unpool7 = Unpool3DLayer(gru_s) conv7a = Conv3DLayer(unpool7, (dcnn_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3 rect7a = LeakyReLU(conv7a) conv7b = Conv3DLayer(rect7a, (dcnn_filters[1], 3, 3, 3)) # 128 x 3 x 3 x 3 rect7 = LeakyReLU(conv7b) res7 = AddLayer(unpool7, rect7) # Residual |=> ----------------------------------------------=V # Unpooling -> deconvolution -> LeakyReLU -> DeConv -> LeakyReLU -> unpool8 = Unpool3DLayer(res7) conv8a = Conv3DLayer(unpool8, (dcnn_filters[2], 3, 3, 3)) # 128 x 3 x 3 x 3 rect8a = LeakyReLU(conv8a) conv8b = Conv3DLayer(rect8a, (dcnn_filters[2], 3, 3, 3)) # 128 x 3 x 3 x 3 rect8 = LeakyReLU(conv8b) res8 = AddLayer(unpool8, rect8) # Residual |=> ----------------------------------------------=V # Unpooling -> deconvolution -> LeakyReLU -> DeConv -> LeakyReLU -> unpool9 = Unpool3DLayer(res8) conv9a = Conv3DLayer(unpool9, (dcnn_filters[3], 3, 3, 3)) # 64 x 3 x 3 x 3 rect9a = LeakyReLU(conv9a) conv9b = Conv3DLayer(rect9a, (dcnn_filters[3], 3, 3, 3)) # 64 x 3 x 3 x 3 rect9 = LeakyReLU(conv9b) conv9c = Conv3DLayer(unpool9, (dcnn_filters[3], 1, 1, 1)) # 64 x 1 x 1 x 1 res9 = AddLayer(conv9c, rect9) # Residual |=> ----------------------------------------------=V # Unpooling -> deconvolution -> LeakyReLU -> DeConv -> LeakyReLU -> conv10a = Conv3DLayer(res9, (dcnn_filters[4], 3, 3, 3)) # 32 x 3 x 3 x 3 rect10a = LeakyReLU(conv10a) conv10b = Conv3DLayer(rect10a, (dcnn_filters[4], 3, 3, 3)) # 32 x 3 x 3 x 3 rect10 = LeakyReLU(conv10b) conv10c = Conv3DLayer(rect10a, (dcnn_filters[4], 3, 3, 3)) # 32 x 3 x 3 x 3 res10 = AddLayer(conv10c, rect10) # Last convolution conv11 = Conv3DLayer(res10, (dcnn_filters[5], 3, 3, 3)) # 2 x 3 x 3 x 3 # Softmax layer softmax_loss = SoftmaxWithLoss3D(conv11.output) ########## End Decoder ######### self.loss = softmax_loss.loss(self.y) self.error = softmax_loss.error(self.y) self.params = get_trainable_params() self.output = softmax_loss.prediction() self.activations = [update_all]
def network_definition(self): # theano.config.compute_test_value = 'warn' self.x = tensor5() self.is_x_tensor4 = False img_w = self.img_w img_h = self.img_h n_gru_vox = 4 # n_vox = self.n_vox n_convfilter = [96, 128, 256, 256, 256, 256] n_fc_filters = [1024] n_deconvfilter = [128, 128, 128, 64, 32, 2] input_shape = (self.batch_size, 3, img_w, img_h) # To define weights, define the network structure first x = InputLayer(input_shape) conv1a = ConvLayer(x, (n_convfilter[0], 7, 7)) conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3)) pool1 = PoolLayer(conv1b) conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3)) conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3)) conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1)) pool2 = PoolLayer(conv2c) conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3)) conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3)) conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1)) pool3 = PoolLayer(conv3b) conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3)) conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3)) pool4 = PoolLayer(conv4b) conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3)) conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3)) conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1)) pool5 = PoolLayer(conv5b) conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3)) conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3)) pool6 = PoolLayer(conv6b) flat6 = FlattenLayer(pool6) fc7 = TensorProductLayer(flat6, n_fc_filters[0]) # Set the size to be 256x4x4x4 s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox) # Dummy 3D grid hidden representations prev_s = InputLayer(s_shape) t_x_s_update = FCConv3DLayer( prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) t_x_s_reset = FCConv3DLayer( prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) reset_gate = SigmoidLayer(t_x_s_reset) rs = EltwiseMultiplyLayer(reset_gate, prev_s) t_x_rs = FCConv3DLayer(rs, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) flat6_ = FlattenLayer(pool6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) prev_s_ = InputLayer(s_shape, prev_s_tensor) t_x_s_update_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_update.params) t_x_s_reset_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_reset.params) update_gate_ = SigmoidLayer(t_x_s_update_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) t_x_rs_ = FCConv3DLayer( rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_.output, update_gate_.output s_update, _ = theano.scan( recurrence, sequences=[ self.x ], # along with images, feed in the index of the current frame outputs_info=[ tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX), tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX) ]) update_all = s_update[-1] s_all = s_update[0] s_last = s_all[-1] gru_s = InputLayer(s_shape, s_last) unpool7 = Unpool3DLayer(gru_s) conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3)) rect7a = LeakyReLU(conv7a) conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3)) rect7 = LeakyReLU(conv7b) res7 = AddLayer(unpool7, rect7) unpool8 = Unpool3DLayer(res7) conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3)) rect8a = LeakyReLU(conv8a) conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3)) rect8 = LeakyReLU(conv8b) res8 = AddLayer(unpool8, rect8) # unpool9 = Unpool3DLayer(res8) conv9a = Conv3DLayer(res8, (n_deconvfilter[3], 3, 3, 3)) rect9a = LeakyReLU(conv9a) conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3)) rect9 = LeakyReLU(conv9b) conv9c = Conv3DLayer(conv9a, (n_deconvfilter[3], 1, 1, 1)) res9 = AddLayer(conv9c, rect9) conv10a = Conv3DLayer(res9, (n_deconvfilter[4], 3, 3, 3)) rect10a = LeakyReLU(conv10a) conv10b = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3)) rect10 = LeakyReLU(conv10b) conv10c = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3)) res10 = AddLayer(conv10c, rect10) conv11 = Conv3DLayer(res10, (n_deconvfilter[5], 3, 3, 3)) #Debug # con12_mat = tensor.matrix(conv12.output) # con12_mat.tag.test_value = np.random.rand(24, 16, 2, 16, 16) softmax_loss = SoftmaxWithLoss3D(conv11.output) self.loss = softmax_loss.loss(self.y) self.error = softmax_loss.error(self.y) self.params = get_trainable_params() self.output = softmax_loss.prediction() self.activations = [update_all]
def recurrence(x_curr, prev_h_tensor, prev_s_tensor): # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) flat6_ = FlattenLayer(pool6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) #LSTM # Dummy 3D grid hidden representations for previous hidden state and cell state prev_h_ = InputLayer(h_shape, prev_h_tensor) prev_s_ = InputLayer(h_shape, prev_s_tensor) t_x_s_forget_ = FCConv3DLayer( prev_h_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_forget.params) t_x_s_input_ = FCConv3DLayer( prev_h_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_input.params) t_x_s_cell_ = FCConv3DLayer( prev_h_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_cell.params) forget_gate_ = SigmoidLayer(t_x_s_forget_) input_gate_ = SigmoidLayer(t_x_s_input_) tanh_t_x_s_cell_ = TanhLayer(t_x_s_cell_) #current cell state cell_state_ = AddLayer( EltwiseMultiplyLayer(forget_gate_, prev_s_), EltwiseMultiplyLayer(input_gate_, tanh_t_x_s_cell_)) #current hidden state, i.e. the output of lstm hidden_state_ = TanhLayer(cell_state_) return hidden_state_.output, cell_state_.output
def network_definition(self): # (multi_views, self.batch_size, 3, self.img_h, self.img_w), self.x = tensor5() self.is_x_tensor4 = False img_w = self.img_w img_h = self.img_h n_gru_vox = 4 # n_vox = self.n_vox n_convfilter = [96, 128, 256, 256, 256, 256] n_fc_filters = [1024] n_deconvfilter = [128, 128, 128, 64, 32, 2] input_shape = (self.batch_size, 3, img_w, img_h) # To define weights, define the network structure first x = InputLayer(input_shape) conv1a = ConvLayer(x, (n_convfilter[0], 7, 7)) conv1b = ConvLayer(conv1a, (n_convfilter[0], 3, 3)) pool1 = PoolLayer(conv1b) conv2a = ConvLayer(pool1, (n_convfilter[1], 3, 3)) conv2b = ConvLayer(conv2a, (n_convfilter[1], 3, 3)) conv2c = ConvLayer(pool1, (n_convfilter[1], 1, 1)) pool2 = PoolLayer(conv2c) conv3a = ConvLayer(pool2, (n_convfilter[2], 3, 3)) conv3b = ConvLayer(conv3a, (n_convfilter[2], 3, 3)) conv3c = ConvLayer(pool2, (n_convfilter[2], 1, 1)) pool3 = PoolLayer(conv3b) conv4a = ConvLayer(pool3, (n_convfilter[3], 3, 3)) conv4b = ConvLayer(conv4a, (n_convfilter[3], 3, 3)) pool4 = PoolLayer(conv4b) conv5a = ConvLayer(pool4, (n_convfilter[4], 3, 3)) conv5b = ConvLayer(conv5a, (n_convfilter[4], 3, 3)) conv5c = ConvLayer(pool4, (n_convfilter[4], 1, 1)) pool5 = PoolLayer(conv5b) conv6a = ConvLayer(pool5, (n_convfilter[5], 3, 3)) conv6b = ConvLayer(conv6a, (n_convfilter[5], 3, 3)) pool6 = PoolLayer(conv6b) #pool6.output_shape[1] = 0 flat6 = FlattenLayer(pool6) #flat6.output_shape[1] = 0 fc7 = TensorProductLayer(flat6, n_fc_filters[0]) # Set the size to be 256x4x4x4 s_shape = (self.batch_size, n_gru_vox, n_deconvfilter[0], n_gru_vox, n_gru_vox) # Dummy 3D grid hidden representations prev_s = InputLayer(s_shape) t_x_s_update = FCConv3DLayer(prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) t_x_s_reset = FCConv3DLayer(prev_s, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) reset_gate = SigmoidLayer(t_x_s_reset) rs = EltwiseMultiplyLayer(reset_gate, prev_s) t_x_rs = FCConv3DLayer(rs, fc7, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3)) def recurrence(x_curr, prev_s_tensor, prev_in_gate_tensor): # Scan function cannot use compiled function. input_ = InputLayer(input_shape, x_curr) conv1a_ = ConvLayer(input_, (n_convfilter[0], 7, 7), params=conv1a.params) rect1a_ = LeakyReLU(conv1a_) conv1b_ = ConvLayer(rect1a_, (n_convfilter[0], 3, 3), params=conv1b.params) rect1_ = LeakyReLU(conv1b_) pool1_ = PoolLayer(rect1_) conv2a_ = ConvLayer(pool1_, (n_convfilter[1], 3, 3), params=conv2a.params) rect2a_ = LeakyReLU(conv2a_) conv2b_ = ConvLayer(rect2a_, (n_convfilter[1], 3, 3), params=conv2b.params) rect2_ = LeakyReLU(conv2b_) conv2c_ = ConvLayer(pool1_, (n_convfilter[1], 1, 1), params=conv2c.params) res2_ = AddLayer(conv2c_, rect2_) pool2_ = PoolLayer(res2_) conv3a_ = ConvLayer(pool2_, (n_convfilter[2], 3, 3), params=conv3a.params) rect3a_ = LeakyReLU(conv3a_) conv3b_ = ConvLayer(rect3a_, (n_convfilter[2], 3, 3), params=conv3b.params) rect3_ = LeakyReLU(conv3b_) conv3c_ = ConvLayer(pool2_, (n_convfilter[2], 1, 1), params=conv3c.params) res3_ = AddLayer(conv3c_, rect3_) pool3_ = PoolLayer(res3_) conv4a_ = ConvLayer(pool3_, (n_convfilter[3], 3, 3), params=conv4a.params) rect4a_ = LeakyReLU(conv4a_) conv4b_ = ConvLayer(rect4a_, (n_convfilter[3], 3, 3), params=conv4b.params) rect4_ = LeakyReLU(conv4b_) pool4_ = PoolLayer(rect4_) conv5a_ = ConvLayer(pool4_, (n_convfilter[4], 3, 3), params=conv5a.params) rect5a_ = LeakyReLU(conv5a_) conv5b_ = ConvLayer(rect5a_, (n_convfilter[4], 3, 3), params=conv5b.params) rect5_ = LeakyReLU(conv5b_) conv5c_ = ConvLayer(pool4_, (n_convfilter[4], 1, 1), params=conv5c.params) res5_ = AddLayer(conv5c_, rect5_) pool5_ = PoolLayer(res5_) conv6a_ = ConvLayer(pool5_, (n_convfilter[5], 3, 3), params=conv6a.params) rect6a_ = LeakyReLU(conv6a_) conv6b_ = ConvLayer(rect6a_, (n_convfilter[5], 3, 3), params=conv6b.params) rect6_ = LeakyReLU(conv6b_) res6_ = AddLayer(pool5_, rect6_) pool6_ = PoolLayer(res6_) flat6_ = FlattenLayer(pool6_) fc7_ = TensorProductLayer(flat6_, n_fc_filters[0], params=fc7.params) rect7_ = LeakyReLU(fc7_) prev_s_ = InputLayer(s_shape, prev_s_tensor) t_x_s_update_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_update.params) t_x_s_reset_ = FCConv3DLayer( prev_s_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_s_reset.params) update_gate_ = SigmoidLayer(t_x_s_update_) comp_update_gate_ = ComplementLayer(update_gate_) reset_gate_ = SigmoidLayer(t_x_s_reset_) rs_ = EltwiseMultiplyLayer(reset_gate_, prev_s_) t_x_rs_ = FCConv3DLayer( rs_, rect7_, (n_deconvfilter[0], n_deconvfilter[0], 3, 3, 3), params=t_x_rs.params) tanh_t_x_rs_ = TanhLayer(t_x_rs_) gru_out_ = AddLayer( EltwiseMultiplyLayer(update_gate_, prev_s_), EltwiseMultiplyLayer(comp_update_gate_, tanh_t_x_rs_)) return gru_out_.output, update_gate_.output s_update, _ = theano.scan(recurrence, sequences=[self.x], # along with images, feed in the index of the current frame outputs_info=[tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX), tensor.zeros_like(np.zeros(s_shape), dtype=theano.config.floatX)]) # -----changes applied----------- # with tensor.S.Session() # pool5.output_shape[1] = 1 '''print("This is updated list function") #theano.printing.pydotprint(s_update) print(*s_update) print(type(s_update[-2])) temp = s_update[0] #print(temp.eval()) #pp(s_update.maker.fgraph.outputs[0]) #s_update[0] #tensor.mul(s_update[-1], 3) print(tensor.maximum(s_update[0],s_update[1]))''' z1 = s_update print("This is z1", z1[0]) z1 = theano.tensor.dot(z1[0], 5) temp = theano.tensor.dot(s_update[0], 5) ############################################### #z2 = theano.tensor.stack([z1[0],temp[0]],2) update_all = z1[-1] print("This is update_all", update_all) s_all = z1[0] s_last = s_all[-1] print("This is s_last shape", type(s_last)) #print("This is the value of s_last:", s_last.eval()) #print(s_last.eval(-1)) gru_s = InputLayer(s_shape, s_last) unpool7 = Unpool3DLayer(gru_s) conv7a = Conv3DLayer(unpool7, (n_deconvfilter[1], 3, 3, 3)) rect7a = LeakyReLU(conv7a) conv7b = Conv3DLayer(rect7a, (n_deconvfilter[1], 3, 3, 3)) rect7 = LeakyReLU(conv7b) res7 = AddLayer(unpool7, rect7) unpool8 = Unpool3DLayer(res7) conv8a = Conv3DLayer(unpool8, (n_deconvfilter[2], 3, 3, 3)) rect8a = LeakyReLU(conv8a) conv8b = Conv3DLayer(rect8a, (n_deconvfilter[2], 3, 3, 3)) rect8 = LeakyReLU(conv8b) res8 = AddLayer(unpool8, rect8) unpool9 = Unpool3DLayer(res8) conv9a = Conv3DLayer(unpool9, (n_deconvfilter[3], 3, 3, 3)) rect9a = LeakyReLU(conv9a) conv9b = Conv3DLayer(rect9a, (n_deconvfilter[3], 3, 3, 3)) rect9 = LeakyReLU(conv9b) conv9c = Conv3DLayer(unpool9, (n_deconvfilter[3], 1, 1, 1)) res9 = AddLayer(conv9c, rect9) conv10a = Conv3DLayer(res9, (n_deconvfilter[4], 3, 3, 3)) rect10a = LeakyReLU(conv10a) conv10b = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3)) rect10 = LeakyReLU(conv10b) conv10c = Conv3DLayer(rect10a, (n_deconvfilter[4], 3, 3, 3)) res10 = AddLayer(conv10c, rect10) conv11 = Conv3DLayer(res10, (n_deconvfilter[5], 3, 3, 3)) softmax_loss = SoftmaxWithLoss3D(conv11.output) self.loss = softmax_loss.loss(self.y) self.error = softmax_loss.error(self.y) self.params = get_trainable_params() self.output = softmax_loss.prediction() update_all = theano.tensor.dot(update_all[0], 5) self.activations = [update_all]