def BaseCapsNet(input_shape, n_class, routings): # assemble encoder x = Input(shape=input_shape) l = x l = Conv2D(256, (9, 9), strides=(2, 2), activation='relu', padding="same")(l) l = BatchNormalization()(l) l = Conv2D(256, (9, 9), strides=(2, 2), activation='relu', padding="same")(l) l = BatchNormalization()(l) l = ConvertToCaps()(l) l = Conv2DCaps(16, 6, kernel_size=(3, 3), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1])(l) l = FlattenCaps()(l) digits_caps = CapsuleLayer(num_capsule=10, dim_capsule=8, routings=routings, channels=0, name='digit_caps')(l) l = CapsToScalars(name='capsnet')(digits_caps) m_capsnet = models.Model(inputs=x, outputs=l, name='capsnet_model') y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digits_caps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask()(digits_caps) # Decoder Network decoder = models.Sequential(name='decoder') decoder.add(Dense(input_dim=80, activation="relu", output_dim=8 * 8 * 16)) decoder.add(Reshape((8, 8, 16))) decoder.add(BatchNormalization(momentum=0.8)) decoder.add( layers.Deconvolution2D(64, 3, 3, subsample=(1, 1), border_mode='same')) decoder.add( layers.Deconvolution2D(32, 3, 3, subsample=(2, 2), border_mode='same')) decoder.add( layers.Deconvolution2D(16, 3, 3, subsample=(2, 2), border_mode='same')) decoder.add( layers.Deconvolution2D(3, 3, 3, subsample=(1, 1), border_mode='same')) decoder.add(Activation("relu")) decoder.add(layers.Reshape(target_shape=(32, 32, 3), name='out_recon')) train_model = models.Model( [x, y], [m_capsnet.output, decoder(masked_by_y)]) eval_model = models.Model(x, [m_capsnet.output, decoder(masked)]) train_model.summary() return train_model, eval_model
def PureCapsNet(input_shape, n_class, routings): # K.set_image_dim_ordering('th') x = Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = Conv2D(filters=128, kernel_size=3, strides=2, padding='valid', activation='relu', name='conv1')(x) conv2 = Conv2D(filters=128, kernel_size=3, strides=2, padding='valid', activation='relu', name='conv2')(conv1) conv3 = Conv2D(filters=128, kernel_size=3, strides=2, padding='valid', activation='relu', name='conv3')(conv2) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv3, dim_capsule=C.DIM_CAPSULE, n_channels=16, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=C.DIM_CAPSULE, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Models for training and evaluation (prediction) train_model = models.Model(x, out_caps, name='PureCapsNet') return train_model
def DeepCapsNetTwoPath(input_shape1,input_shape2, n_class, routings): # assemble encoder x33 = Input(shape=input_shape1) x15 = Input(shape=input_shape2,name='2_0') path1 = Conv2D(128, (5, 5), strides=(1, 1), activation='relu', padding="same",name='1_1')(x33) # common conv layer path1=MaxPooling2D(pool_size=(2, 2),name='1_2')(path1) path1=Dropout(0.5)(path1) path1 = ConvertToCaps()(path1) path1 = Conv2DCaps(16, 8, kernel_size=(5, 5), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1],name='1_3')(path1) path1 = Conv2DCaps(16, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1],name='1_4')(path1) path2 = Conv2D(128, (5, 5), strides=(1, 1), activation='relu', padding="same",name='2_1')(x15) # common conv layer path2 = ConvertToCaps()(path2) path2 = Conv2DCaps(16, 8, kernel_size=(5, 5), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1],name='2_2')(path2) path2 = Conv2DCaps(16, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1],name='2_3')(path2) x = layers.Add()([path1, path2]) l=x # l = Conv2D(128, (5, 5), strides=(1, 1), activation='relu', padding="same")(l) # common conv layer # # l = BatchNormalization()(l) # l = ConvertToCaps()(l) # l = Conv2DCaps(32, 4, kernel_size=(3, 3), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1])(l) # l_skip = Conv2DCaps(32, 4, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) # l = Conv2DCaps(32, 4, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) # l = Conv2DCaps(32, 4, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) # l = layers.Add()([l, l_skip]) l = Conv2DCaps(16, 8, kernel_size=(5, 5), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1],name='3_1')(l) l_skip = Conv2DCaps(16, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1],name='3_2')(l) l = Conv2DCaps(16, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1],name='3_3')(l) l = layers.Add()([l, l_skip]) # l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1])(l) # l_skip = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) # l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) # l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) # l = layers.Add()([l, l_skip]) # l1 = l # l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1])(l) # l_skip = ConvCapsuleLayer3D(kernel_size=3, num_capsule=32, num_atoms=8, strides=1, padding='same', routings=3)(l) # l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) # l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) # l = layers.Add()([l, l_skip]) # l2 = l la = FlattenCaps()(l) # la = FlattenCaps()(l2) # lb = FlattenCaps()(l1) # l = layers.Concatenate(axis=-2)([la, lb]) # l = Dropout(0.4)(l) digits_caps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, channels=0, name='digit_caps')(la) l = CapsToScalars(name='capsnet')(digits_caps) m_capsnet = models.Model(inputs=[x33,x15], outputs=l, name='capsnet_model') y = Input(shape=(n_class,)) masked_by_y = Mask_CID()([digits_caps, y]) masked = Mask_CID()(digits_caps) # Decoder Network decoder = models.Sequential(name='decoder') decoder.add(Dense(input_dim=16, activation="relu", output_dim=8 *8 * 16)) decoder.add(Reshape((8, 8, 16))) decoder.add(BatchNormalization(momentum=0.8)) decoder.add(Deconvolution2D(64, 3, 3, subsample=(1, 1), border_mode='same')) decoder.add(Deconvolution2D(32, 3, 3, subsample=(2, 2), border_mode='same')) decoder.add(Deconvolution2D(16, 3, 3, subsample=(1, 1), border_mode='same')) decoder.add(Deconvolution2D(1, 3, 3, subsample=(2, 2), border_mode='valid')) decoder.add(Activation("relu")) decoder.add(Reshape(target_shape=(33, 33, 1), name='out_recon')) train_model= models.Model([x33,x15, y], [m_capsnet.output, decoder(masked_by_y)]) eval_model = models.Model([x33,x15], [m_capsnet.output, decoder(masked)]) train_model.summary() return train_model, eval_model
def MsECapsNet(input_shape, n_class, routings): # K.set_image_dim_ordering('th') x = Input(input_shape, name='input') # Part1 conv1 = Conv2D(64, (3, 3), padding='same', name='conv1')(x) bn1 = BatchNormalization(name='bn1')(conv1) relu1 = ReLU()(bn1) pool1 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool1')(relu1) aside_pool1 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='aside_pool1')(x) concat1 = concatenate([pool1, aside_pool1]) conv2 = Conv2D(128, (3, 3), padding='same', name='conv2')(pool1) bn2 = BatchNormalization(name='bn2')(conv2) relu2 = ReLU()(bn2) pool2 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool2')(relu2) aside_pool2 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='aside_pool2')(concat1) concat2 = concatenate([pool2, aside_pool2]) conv3 = Conv2D(128, (3, 3), padding='same', name='conv3')(pool2) bn3 = BatchNormalization(name='bn3')(conv3) relu3 = ReLU()(bn3) pool3 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool3')(relu3) aside_pool3 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='aside_pool3')(concat2) concat3 = concatenate([pool3, aside_pool3]) conv4 = Conv2D(128, (3, 3), padding='same', name='conv4')(pool3) bn4 = BatchNormalization(name='bn4')(conv4) relu4 = ReLU()(bn4) pool4 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool4')(relu4) aside_pool4 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='aside_pool4')(concat3) concat4 = concatenate([pool4, aside_pool4]) conv5 = Conv2D(128, (3, 3), padding='same', name='conv5')(pool4) bn5 = BatchNormalization(name='bn5')(conv5) relu5 = ReLU()(bn5) pool5 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool5')(relu5) aside_pool5 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='aside_pool5')(concat4) concat5 = concatenate([pool5, aside_pool5]) # Part2-branch-a primarycaps = PrimaryCap(concat5, dim_capsule=8, n_channels=32, kernel_size=3, strides=2, padding='same', name='primarycaps') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) # Part2-branch-b flastten = Flatten()(concat5) fc1 = Dense(n_class, activation='sigmoid', name='fc1')(flastten) # Part3 add = Add(name='add')([out_caps, fc1]) train_model = models.Model(x, add, name='MsECapsNet') return train_model
def DenseCapsNet(input_shape, n_class, routings): # K.set_image_dim_ordering('th') x = Input(input_shape, name='input') # Part1 conv1 = Conv2D(64, (3, 3), padding='same', name='conv1')(x) bn1 = BatchNormalization(name='bn1')(conv1) relu1 = ReLU()(bn1) pool1 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool1')(relu1) drop1 = Dropout(0.3, name='dropout1')(pool1) conv2 = Conv2D(128, (3, 3), padding='same', name='conv2')(drop1) bn2 = BatchNormalization(name='bn2')(conv2) relu2 = ReLU()(bn2) pool2 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool2')(relu2) drop2 = Dropout(0.3, name='dropout2')(pool2) conv3 = Conv2D(128, (3, 3), padding='same', name='conv3')(drop2) bn3 = BatchNormalization(name='bn3')(conv3) relu3 = ReLU()(bn3) pool3 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool3')(relu3) drop3 = Dropout(0.3, name='dropout3')(pool3) conv4 = Conv2D(128, (3, 3), padding='same', name='conv4')(drop3) bn4 = BatchNormalization(name='bn4')(conv4) relu4 = ReLU()(bn4) pool4 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool4')(relu4) drop4 = Dropout(0.3, name='dropout4')(pool4) conv5 = Conv2D(128, (3, 3), padding='same', name='conv5')(drop4) bn5 = BatchNormalization(name='bn5')(conv5) relu5 = ReLU()(bn5) pool5 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool5')(relu5) drop5 = Dropout(0.3, name='dropout5')(pool5) conv6 = Conv2D(128, (3, 3), padding='same', name='conv6')(drop5) bn6 = BatchNormalization(name='bn6')(conv6) relu6 = ReLU()(bn6) pool6 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool6')(relu6) drop6 = Dropout(0.3, name='dropout6')(pool6) conv7 = Conv2D(128, (3, 3), padding='same', name='conv7')(drop6) bn7 = BatchNormalization(name='bn7')(conv7) relu7 = ReLU()(bn7) pool7 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool7')(relu7) drop7 = Dropout(0.3, name='dropout7')(pool7) # Part2-branch-a primarycaps = PrimaryCap(drop6, dim_capsule=8, n_channels=32, kernel_size=3, strides=2, padding='same', name='primarycaps') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) # # Part2-branch-b densenet = DenseNet(img_input=drop2, classes=50, activation='sigmoid', depth=40) # # Part3 add = Add(name='add')([out_caps, densenet]) train_model = models.Model(x, add, name='DenseCapsNet') return train_model
def MultiScaleCapsNet(input_shape, n_class, routings): # K.set_image_dim_ordering('th') x = Input(input_shape, name='input') # Part1 conv1 = Conv2D(64, (3, 3), padding='same', name='conv1')(x) bn1 = BatchNormalization(name='bn1')(conv1) relu1 = ReLU()(bn1) pool1 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool1')(relu1) drop1 = Dropout(0.3, name='dropout1')(pool1) conv2 = Conv2D(128, (3, 3), padding='same', name='conv2')(drop1) bn2 = BatchNormalization(name='bn2')(conv2) relu2 = ReLU()(bn2) pool2 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool2')(relu2) drop2 = Dropout(0.3, name='dropout2')(pool2) conv3 = Conv2D(128, (3, 3), padding='same', name='conv3')(drop2) bn3 = BatchNormalization(name='bn3')(conv3) relu3 = ReLU()(bn3) pool3 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool3')(relu3) drop3 = Dropout(0.3, name='dropout3')(pool3) conv4 = Conv2D(128, (3, 3), padding='same', name='conv4')(drop3) bn4 = BatchNormalization(name='bn4')(conv4) relu4 = ReLU()(bn4) pool4 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool4')(relu4) drop4 = Dropout(0.3, name='dropout4')(pool4) conv5 = Conv2D(128, (3, 3), padding='same', name='conv5')(drop4) bn5 = BatchNormalization(name='bn5')(conv5) relu5 = ReLU()(bn5) pool5 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool5')(relu5) drop5 = Dropout(0.3, name='dropout5')(pool5) conv6 = Conv2D(128, (3, 3), padding='same', name='conv6')(drop5) bn6 = BatchNormalization(name='bn6')(conv6) relu6 = ReLU()(bn6) pool6 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool6')(relu6) drop6 = Dropout(0.3, name='dropout6')(pool6) conv7 = Conv2D(128, (3, 3), padding='same', name='conv7')(drop6) bn7 = BatchNormalization(name='bn7')(conv7) relu7 = ReLU()(bn7) pool7 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool7')(relu7) drop7 = Dropout(0.3, name='dropout7')(pool7) conv8 = Conv2D(128, (3, 3), padding='same', name='conv8')(drop7) bn8 = BatchNormalization(name='bn8')(conv8) relu8 = ReLU()(bn8) pool8 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool8')(relu8) drop8 = Dropout(0.3, name='dropout8')(pool8) # Part2-branch-a primarycaps6 = PrimaryCap(drop6, dim_capsule=8, n_channels=16, kernel_size=3, strides=2, padding='same', name='primarycaps6') digitcaps6 = CapsuleLayer(num_capsule=n_class, dim_capsule=32, routings=routings, name='digitcaps6')(primarycaps6) out_caps6 = Length(name='capsnet6')(digitcaps6) primarycaps7 = PrimaryCap(drop7, dim_capsule=8, n_channels=16, kernel_size=3, strides=2, padding='same', name='primarycaps7') digitcaps7 = CapsuleLayer(num_capsule=n_class, dim_capsule=32, routings=routings, name='digitcaps7')(primarycaps7) out_caps7 = Length(name='capsnet7')(digitcaps7) primarycaps8 = PrimaryCap(drop8, dim_capsule=8, n_channels=16, kernel_size=3, strides=2, padding='same', name='primarycaps8') digitcaps8 = CapsuleLayer(num_capsule=n_class, dim_capsule=32, routings=routings, name='digitcaps8')(primarycaps8) out_caps8 = Length(name='capsnet8')(digitcaps8) # Part2-branch-b flastten = Flatten()(drop7) fc1 = Dense(n_class, activation='sigmoid', name='fc1')(flastten) # Part3 add = Add(name='add')([out_caps6, out_caps7, out_caps8, fc1]) train_model = models.Model(x, add, name='MultiScaleCapsNet') return train_model
def TestMixCapsNet(input_shape, n_class, routings): # K.set_image_dim_ordering('th') x = Input(input_shape, name='input') # Part1 conv1 = Conv2D(64, (3, 3), padding='same', name='conv1')(x) bn1 = BatchNormalization(name='bn1')(conv1) relu1 = ReLU()(bn1) pool1 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool1')(relu1) drop1 = Dropout(0.3, name='dropout1')(pool1) # cbam1 = cbam_block(drop1, 8) conv2 = Conv2D(128, (3, 3), padding='same', name='conv2')(drop1) # conv2 = Conv2D(128, (3, 3), padding='same', name='conv2')(cbam1) bn2 = BatchNormalization(name='bn2')(conv2) relu2 = ReLU()(bn2) pool2 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool2')(relu2) drop2 = Dropout(0.3, name='dropout2')(pool2) # cbam2 = cbam_block(drop2, 8) conv3 = Conv2D(128, (3, 3), padding='same', name='conv3')(drop2) # conv3 = Conv2D(128, (3, 3), padding='same', name='conv3')(cbam2) bn3 = BatchNormalization(name='bn3')(conv3) relu3 = ReLU()(bn3) pool3 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool3')(relu3) drop3 = Dropout(0.3, name='dropout3')(pool3) cbam3 = cbam_block(drop3, 8) conv4 = Conv2D(128, (3, 3), padding='same', name='conv4')(drop3) # conv4 = Conv2D(128, (3, 3), padding='same', name='conv4')(cbam3) bn4 = BatchNormalization(name='bn4')(conv4) relu4 = ReLU()(bn4) pool4 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool4')(relu4) drop4 = Dropout(0.3, name='dropout4')(pool4) # cbam4 = cbam_block(drop4, 8) conv5 = Conv2D(128, (3, 3), padding='same', name='conv5')(drop4) # conv5 = Conv2D(128, (3, 3), padding='same', name='conv5')(cbam4) bn5 = BatchNormalization(name='bn5')(conv5) relu5 = ReLU()(bn5) pool5 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool5')(relu5) drop5 = Dropout(0.3, name='dropout5')(pool5) # cbam5 = cbam_block(drop5, 8) conv6 = Conv2D(128, (3, 3), padding='same', name='conv6')(drop5) # conv6 = Conv2D(128, (3, 3), padding='same', name='conv6')(cbam5) bn6 = BatchNormalization(name='bn6')(conv6) relu6 = ReLU()(bn6) pool6 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool6')(relu6) drop6 = Dropout(0.3, name='dropout6')(pool6) # cbam6 = cbam_block(drop6, 8) conv7 = Conv2D(128, (3, 3), padding='same', name='conv7')(drop6) # conv7 = Conv2D(128, (3, 3), padding='same', name='conv7')(cbam6) bn7 = BatchNormalization(name='bn7')(conv7) relu7 = ReLU()(bn7) pool7 = MaxPool2D((2, 2), strides=(2, 2), padding='same', name='pool7')(relu7) drop7 = Dropout(0.3, name='dropout7')(pool7) # cbam7 = cbam_block(drop7, 8) # Part2-branch-a primarycaps = PrimaryCap(drop7, dim_capsule=8, n_channels=32, kernel_size=3, strides=2, padding='same', name='primarycaps') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) # fc1 = Dense(n_class, activation='sigmoid', name='fc1')(out_caps) # Part2-branch-b flastten = Flatten()(drop7) fc1 = Dense(n_class, activation='sigmoid', name='fc1')(flastten) # flastten = Flatten()(drop7) # fc1 = Dense(128, activation='relu', name='fc1')(flastten) # fc2 = Dense(n_class, activation='sigmoid', name='fc2')(fc1) # timedis = TimeDistributed(Flatten(), name='timedis')(drop7) # gru1 = LSTM(64, return_sequences=True, name='gru1')(timedis) # gru2 = LSTM(64, return_sequences=False, name='gru2')(gru1) # fc1 = Dense(n_class, activation='sigmoid', name='fc1')(gru2) # Part3 add = Add(name='add')([out_caps, fc1]) train_model = models.Model(x, add, name='TestMixCapsNet') return train_model
def DeepCapsNet(input_shape, n_class, routings): # assemble encoder x = Input(shape=input_shape) l = x l = Conv2D(128, (3, 3), strides=(1, 1), activation='relu', padding="same")(l) # common conv layer l = BatchNormalization()(l) l = ConvertToCaps()(l) l = Conv2DCaps(32, 4, kernel_size=(3, 3), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1])(l) l_skip = Conv2DCaps(32, 4, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = Conv2DCaps(32, 4, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = Conv2DCaps(32, 4, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = layers.Add()([l, l_skip]) l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1])(l) l_skip = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = layers.Add()([l, l_skip]) l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1])(l) l_skip = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = layers.Add()([l, l_skip]) l1 = l l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(2, 2), r_num=1, b_alphas=[1, 1, 1])(l) l_skip = ConvCapsuleLayer3D(kernel_size=3, num_capsule=32, num_atoms=8, strides=1, padding='same', routings=3)(l) l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = Conv2DCaps(32, 8, kernel_size=(3, 3), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(l) l = layers.Add()([l, l_skip]) l2 = l la = FlattenCaps()(l2) lb = FlattenCaps()(l1) l = layers.Concatenate(axis=-2)([la, lb]) # l = Dropout(0.4)(l) digits_caps = CapsuleLayer(num_capsule=n_class, dim_capsule=32, routings=routings, channels=0, name='digit_caps')(l) l = CapsToScalars(name='capsnet')(digits_caps) m_capsnet = models.Model(inputs=x, outputs=l, name='capsnet_model') y = Input(shape=(n_class, )) masked_by_y = Mask_CID()([digits_caps, y]) masked = Mask_CID()(digits_caps) # Decoder Network decoder = models.Sequential(name='decoder') decoder.add(Dense(input_dim=32, activation="relu", output_dim=8 * 8 * 16)) decoder.add(Reshape((8, 8, 16))) decoder.add(BatchNormalization(momentum=0.8)) decoder.add(Deconvolution2D(64, 3, 3, subsample=(1, 1), border_mode='same')) decoder.add(Deconvolution2D(32, 3, 3, subsample=(2, 2), border_mode='same')) decoder.add(Deconvolution2D(16, 3, 3, subsample=(2, 2), border_mode='same')) decoder.add(Deconvolution2D(8, 3, 3, subsample=(2, 2), border_mode='same')) decoder.add(Deconvolution2D(3, 3, 3, subsample=(1, 1), border_mode='same')) decoder.add(Activation("relu")) decoder.add(Reshape(target_shape=(64, 64, 3), name='out_recon')) train_model = models.Model( [x, y], [m_capsnet.output, decoder(masked_by_y)]) eval_model = models.Model(x, [m_capsnet.output, decoder(masked)]) train_model.summary() return train_model, eval_model