def make_model(): x = Input(shape=input_shape) conv1 = Conv3D(filters=conv_layer_filters, kernel_size=first_layer_kernel_size, strides=1, padding='valid', activation='relu', name='conv1')(x) primarycaps = PrimaryCap(conv1, dim_capsule=dim_primary_capsule, n_channels=n_channels, kernel_size=primary_cap_kernel_size, strides=2, padding='valid', name='primarycap_conv3d') sub_caps = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_sub_capsule, routings=3, name='sub_caps')(primarycaps) out_caps = Length(name='capsnet')(sub_caps) # Decoder network y = Input(shape=(n_class,)) masked_by_y = Mask()([sub_caps, y]) masked = Mask()(sub_caps) # shared decoder model in training and prediction decoder = Sequential(name='decoder') decoder.add(Dense(512, activation='relu', input_dim=dim_sub_capsule*n_class)) decoder.add(Dense(1024, activation='relu')) decoder.add(Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(Reshape(target_shape=input_shape, name='out_recon')) ### Models for training and evaluation (prediction and actually using) train_model = Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = Model(x, [out_caps, decoder(masked)]) ### manipulate model can be used to visualize activation maps for specific classes noise = Input(shape=(n_class, dim_sub_capsule)) noised_sub_caps = Add()([sub_caps, noise]) masked_noised_y = Mask()([noised_sub_caps, y]) manipulate_model = Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, num_routing): """ MNIST Veriseti için Kapsül Ağları. :param input_shape: veri şekli, 3d, [genişlik, yükseklik, kanal] :param n_class: sınıf sayısı :param num_routing: dinamik yönlendirme (dynamic routing) iterasyon sayısı :return: iki Keras model, birincisi eğitim için, ikincisi değerlendirme için (evalaution). `eval_model` aynı zamanda eğitim için de kullanılabilir. """ x = layers.Input(shape=input_shape) # Katman 1: Geleneksel Evrişimli Sinir Ağı Katmanı (Conv2D) conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Katman 2: Conv2D katmanı ile ezme (squash) aktivasyonu, [None, num_capsule, dim_capsule]’e yeniden şekil veriliyor. primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Katman 3: Kapsül Katmanı. Dinamik Yönlendirme algoritması burada çalışıyor. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Katman 4: Her kapsülün uzunluğunu yeniden düzenleyen yardımcı bir katmandır. Doğru etiketle eşleşmesi için bu işlem yapılır. # Eğer Tensorflow kullanıyorsanız bu işleme gerek yoktur. :) out_caps = Length(name='capsnet')(digitcaps) # Kodçözücü Ağ. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # Doğru etiket, kapsül katmanın çıkışını maskelemek için kullanılır (Eğitim için). masked = Mask()( digitcaps ) # Filtre (maske), kapsülün maksimal uzunluğu ile kullanılır (Kestirim için). # Eğitim ve Kestirimde Kodçözücü Modelin Paylaşımı decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Eğitim ve Değerlendirme (Kestirim) için Modeller train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet(input_shape, n_class, num_routing): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param num_routing: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction other_masks = [] # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet(input_shape, n_class, routings): x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) # YES decoder.add(layers.Dense(1024, activation='relu')) # YES decoder.add(layers.Dense(4096, activation='relu')) # YES decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings): x = layers.Input(shape=input_shape) # Layers 1: Convolution Layers to extract low level features conv1 = layers.Conv2D(filters=64, kernel_size=3, strides=2, activation='relu', name='Conv1')(x) conv2 = layers.Conv2D(filters=128, kernel_size=3, strides=2, activation='relu', name='Conv2')(conv1) conv3 = layers.Conv2D(filters=256, kernel_size=6, strides=2, activation='relu', name='Conv3')(conv2) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv3, dim_capsule=8, n_channels=32, kernel_size=6, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. classcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='classcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. out_caps = Length(name='capsnet')(classcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()([classcaps, y]) masked = Mask()(classcaps) # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet(input_shape, n_class, routings): """ 针对 MNIST 手写字符识别的胶囊网络 ## input_shape: 输入数据的维度, 长度为3的列表, [width, height, channels] ## n_class: 类别的数量 ## routings: routing算法迭代的次数 :return: 返回两个模型, 第一个用于训练, 第二个用于测试 """ # Encoder network.输入为图片 x = layers.Input(shape=input_shape) # Layer 1: 卷积层 conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Layer 2: PrimaryCap(自定义层) primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: CapsuleLayer(自定义胶囊层) digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # Layer 4: 模值计算层(自定义层) out_caps = Length(name='capsnet')(digitcaps) # Decoder network.输入为向量 y = layers.Input(shape=(n_class, )) # 蒙版操作,对decoder输入的标准化 masked_by_y = Mask()([digitcaps, y]) # 用真实值取代胶囊层的输出值。此项用于训练 masked = Mask()(digitcaps) # 用最大长度值做胶囊的蒙版。此项用于评估 # 包含3层的全连接神经网络(序贯模型) decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # 最终将输出转为图片 # 训练模型和评估模型 train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet(input_shape, n_class, num_routing): """ A Capsule Network on MNIST. :param input_shape: data shape, 4d, [None, width, height, channels] :param n_class: number of classes :param num_routing: number of routing iterations :return: A Keras Model with 2 inputs and 2 outputs """ x = layers.Input(shape=(maxlen,)) embed = layers.Embedding(max_features, embed_dim, input_length=maxlen)(x) conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')( embed) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector] primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='out_caps')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class,)) masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. x_recon = layers.Dense(512, activation='relu')(masked) x_recon = layers.Dense(1024, activation='relu')(x_recon) x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon) # two-input-two-output keras Model return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, num_routing): x = layers.Input(shape=input_shape) conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu')(x) primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing)(primarycaps) out_caps = Length(name='out_caps')(digitcaps) # Reconstruction network y = layers.Input(shape=(n_class, )) masked = Mask()([digitcaps, y]) x_recon = layers.Dense(512, activation='relu')(masked) x_recon = layers.Dense(1024, activation='relu')(x_recon) x_recon = layers.Dense(784, activation='sigmoid')(x_recon) x_recon = layers.Reshape(target_shape=[28, 28, 1], name='out_recon')(x_recon) return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, num_routing, kernel_size=7): """ A Capsule Network on 2019ncon. :param input_shape: data shape, 3d, [width, height, channels],for example:[21,28,1] :param n_class: number of classes :param num_routing: number of routing iterations :return: A Keras Model with 2 inputs and 2 outputs """ x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=128, kernel_size=kernel_size, strides=1, padding='same', activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector] primarycaps = PrimaryCap(conv1, dim_vector=16, n_channels=32, kernel_size=kernel_size, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=32, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='out_caps')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class,)) masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. x_recon = layers.Dense(512, activation='relu')(masked) x_recon = layers.Dense(1024, activation='relu')(x_recon) x_recon = layers.Dense(np.prod(input_shape), activation='sigmoid')(x_recon) x_recon = layers.Reshape(target_shape=input_shape, name='out_recon')(x_recon) # two-input-two-output keras Model return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, routings): x = layers.Input(shape=input_shape) conv1 = layers.Conv2D(filters=256, kernel_size=(9), strides=1, padding='valid', activation='relu', name='conv1')(x) primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) y = layers.Input(shape=(n_class, )) masked_by_y = Mask()([digitcaps, y]) masked = Mask()(digitcaps) decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings, batch_size): # input layer x = tf.keras.layers.Input(input_shape, batch_size = batch_size) # layer 1 : regular Convolutionnal layer conv1 = tf.keras.layers.Conv2D(filters = 256, kernel_size = (9,9), activation = 'relu', name = 'conv1')(x) # layer 2 : PrimaryCaps, which is a convolution layer with Squash activation # dim_capsule : corresponds to the dimension of the capsule output vector # n_channels : number of capsule types primarycaps = PrimaryCap(conv1, dim_capsule = 8, n_channels = 32, kernel_size = 9, strides = 2, padding = 'valid') # layer 3 : CapsuleLayer (involves routing by agreement) # each capsule in this layer represents one of the Kuzushiji symbol kcaps = CapsuleLayer(num_capsule = n_class, dim_capsule = 16, routings = routings, name = 'kcaps')(primarycaps) # layer 4 : layer that takes the length of each capsule out_caps = Length(name='capsnet')(kcaps) # Let's build the decoder network # 2 reconstructions are performed : # - first one is to reconstruct image according to the true label # - second one is to reconstruct image according to the vector with maximal length (prediction) y = tf.keras.layers.Input((n_class,)) masked_by_y = Mask()([kcaps, y]) masked = Mask()(kcaps) # Dense layers of the decoder architecture as described in the paper decoder = tf.keras.models.Sequential(name = 'decoder') decoder.add(tf.keras.layers.Dense(512, activation = 'relu', input_dim = 16 * n_class)) decoder.add(tf.keras.layers.Dense(1024, activation = 'relu')) decoder.add(tf.keras.layers.Dense(input_shape[0]*input_shape[1], activation = 'sigmoid')) decoder.add(tf.keras.layers.Reshape(input_shape, name = 'out_recon')) # Models used for training and evaluation # train_model involves training of the decoder # while evaluation model, given an input x, outputs his prediction and his reconstruction using the trained decoder train_model = tf.keras.models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = tf.keras.models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet_nogradientstop_crossentropy(input_shape, n_class, routings): # best testing results! val 0.13xx testX cnn1 200 1 cnn2 150 9 drop1 0.68 drop20.68 n_channels 50 kernel_size 20,dropout1 x = layers.Input(shape=input_shape) conv1 = layers.Conv1D(filters=200, kernel_size=1, strides=1, padding='valid', kernel_initializer='he_normal',activation='relu', name='conv1')(x) #conv1=BatchNormalization()(conv1) conv1 = Dropout(0.7)(conv1) conv2 = layers.Conv1D(filters=200, kernel_size=9, strides=1, padding='valid', kernel_initializer='he_normal',activation='relu', name='conv2')(conv1) #conv1=BatchNormalization()(conv1) conv2 = Dropout(0.75)(conv2) #0.75 valx loss has 0.1278! primarycaps = PrimaryCap(conv2, dim_capsule=8, n_channels=60, kernel_size=20, kernel_initializer='he_normal',strides=1, padding='valid',dropout=0.2) dim_capsule_dim2=10 # Capsule layer. Routing algorithm works here. digitcaps_c = CapsuleLayer_nogradient_stop(num_capsule=n_class, dim_capsule=dim_capsule_dim2, num_routing=routings,name='digitcaps',kernel_initializer='he_normal',dropout=0.1)(primarycaps) #digitcaps_c = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_capsule_dim2, num_routing=routings,name='digitcaps',kernel_initializer='he_normal')(primarycaps) digitcaps = Extract_outputs(dim_capsule_dim2)(digitcaps_c) weight_c = Extract_weight_c(dim_capsule_dim2)(digitcaps_c) out_caps = Length()(digitcaps) out_caps = Activation('softmax',name='capsnet')(out_caps) # Decoder network. y = layers.Input(shape=(n_class,)) masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=dim_capsule_dim2*n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = Model(x, [out_caps, decoder(masked)]) weight_c_model = Model(x,weight_c) # manipulate model noise = layers.Input(shape=(n_class, dim_capsule_dim2)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model,weight_c_model
def CapsNet(input_shape, n_class, routings): """ Defining the CapsNet :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. """ x = layers.Input(shape=input_shape) conv1 = layers.Conv2D(filters=64, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv1')(x) conv2 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv2')(conv1) conv3 = layers.Conv2D(filters=256, kernel_size=3, strides=2, padding='valid', activation='relu', name='conv3')(conv2) primarycaps = PrimaryCap(conv3, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, channels=32, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) """ Decoder Network """ y = layers.Input(shape=(n_class, )) masked_by_y = Mask()([digitcaps, y]) masked = Mask()(digitcaps) decoder = models.Sequential(name='decoder') decoder.add( Dense(input_dim=16 * n_class, activation="relu", output_dim=7 * 7 * 32)) decoder.add(Reshape((7, 7, 32))) decoder.add(BatchNormalization(momentum=0.8)) decoder.add( layers.Deconvolution2D(32, 3, 3, subsample=(1, 1), border_mode='same', activation="relu")) decoder.add( layers.Deconvolution2D(16, 3, 3, subsample=(2, 2), border_mode='same', activation="relu")) decoder.add( layers.Deconvolution2D(8, 3, 3, subsample=(2, 2), border_mode='same', activation="relu")) decoder.add( layers.Deconvolution2D(4, 3, 3, subsample=(1, 1), border_mode='same', activation="relu")) decoder.add( layers.Deconvolution2D(1, 3, 3, subsample=(1, 1), border_mode='same', activation="sigmoid")) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) """ Models for training and evaluation (prediction) """ train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNetBasic(input_shape, n_class=2): x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=5, strides=1, padding='same', activation='relu', name='conv1')(x) # Reshape layer to be 1 capsule x [filters] atoms _, H, W, C = conv1.get_shape() conv1_reshaped = layers.Reshape((H.value, W.value, 1, C.value))(conv1) # Layer 1: Primary Capsule: Conv cap with routing 1 primary_caps = ConvCapsuleLayer(kernel_size=5, num_capsule=8, num_atoms=32, strides=1, padding='same', routings=1, name='primarycaps')(conv1_reshaped) # Layer 4: Convolutional Capsule: 1x1 seg_caps = ConvCapsuleLayer(kernel_size=1, num_capsule=1, num_atoms=16, strides=1, padding='same', routings=3, name='seg_caps')(primary_caps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. out_seg = Length(num_classes=n_class, seg=True, name='out_seg')(seg_caps) # Decoder network. _, H, W, C, A = seg_caps.get_shape() y = layers.Input(shape=input_shape[:-1] + (1, )) masked_by_y = Mask()( [seg_caps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask()( seg_caps) # Mask using the capsule with maximal length. For prediction def shared_decoder(mask_layer): recon_remove_dim = layers.Reshape( (H.value, W.value, A.value))(mask_layer) recon_1 = layers.Conv2D(filters=64, kernel_size=1, padding='same', kernel_initializer='he_normal', activation='relu', name='recon_1')(recon_remove_dim) recon_2 = layers.Conv2D(filters=128, kernel_size=1, padding='same', kernel_initializer='he_normal', activation='relu', name='recon_2')(recon_1) out_recon = layers.Conv2D(filters=1, kernel_size=1, padding='same', kernel_initializer='he_normal', activation='sigmoid', name='out_recon')(recon_2) return out_recon # Models for training and evaluation (prediction) train_model = models.Model(inputs=[x, y], outputs=[out_seg, shared_decoder(masked_by_y)]) eval_model = models.Model(inputs=x, outputs=[out_seg, shared_decoder(masked)]) # manipulate model noise = layers.Input(shape=((H.value, W.value, C.value, A.value))) noised_seg_caps = layers.Add()([seg_caps, noise]) masked_noised_y = Mask()([noised_seg_caps, y]) manipulate_model = models.Model(inputs=[x, y, noise], outputs=shared_decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
kernel_size=9, strides=2, padding="valid") # Capsule Layer with routing algorithm digitcaps = CapsuleLayer(num_capsule=num_classes, dim_capsule=16, routings=num_routing, name="digitcaps")(primarycaps) # Auxilliary layer to replace each capsule with its length out_caps = Length(name="capsnet")(digitcaps) # Building the Decoder Network y = Input(shape=(num_classes, )) masked_by_y = Mask()([digitcaps, y]) # True label used to mask the output masked = Mask()(digitcaps) # Mask using the capsule with maximum length # Shared Decoder model in training and prediction decoder = Sequential(name='decoder') decoder.add(Dense(512, activation="relu", input_dim=16 * num_classes)) decoder.add(Dense(1024, activation="relu")) decoder.add(Dense(np.prod(x_train.shape[1:]), activation="sigmoid")) decoder.add(Reshape(target_shape=input_shape, name="out_recon")) # Models for training model = Model([x, y], [out_caps, decoder(masked_by_y)]) def margin_loss(y_true, y_pred): """
def CapsNet(input_shape, n_class, routings): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ poses_list = [] x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=3, strides=1, padding='same', activation='relu', name='conv1_1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps1 = PrimaryCap1(conv1, dim_capsule=8, n_channels=32, kernel_size=(5, 5), strides=2, padding='same') #y Layer 3: Capsule layer. Routing algorithm works here. digitcaps1 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps1_1')(primarycaps1) poses_list.append(digitcaps1) # Layer 1: Just a conventional Conv2D layer conv2 = layers.Conv2D(filters=256, kernel_size=5, strides=1, padding='same', activation='relu', name='conv1_2')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps2 = PrimaryCap2(conv2, dim_capsule=8, n_channels=32, kernel_size=(5, 5), strides=2, padding='same') print "befor call <=======" # Layer 3: Capsule layer. Routing algorithm works here. digitcaps2 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps1_2')(primarycaps2) print "after call <=======" poses_list.append(digitcaps2) print("before pose[0]:", poses_list[0]) # print("convert_to_tensor:", tf.convert_to_tensor(poses_list)) # poses = tf.reduce_mean(tf.convert_to_tensor(poses_list), axis=0) # print("reduce_mean type: ", type(poses)) # print("reduce_mean : ", poses) # print("pose:", poses) # print("pose[0]:", poses[0]) # pose.layers.average = tf.reduce_mean(poses_list, axis=0) avg = layers.average(poses_list) print("avg:", avg) digitcaps = avg; # digitcaps = K.sqrt(K.sum(K.square(avg), 2, True)) # digitcaps = K.sqrt(K.sum(K.square(avg), 2, True)) print("digitcaps type: ", type(digitcaps)) print("digitcaps: ", digitcaps) # digitcaps = poses; # print("digitcaps1 type: ", type(digitcaps1)) # print("digitcaps1: ", digitcaps1) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) print("out_caps: ", out_caps) # Decoder network. y = layers.Input(shape=(n_class,)) masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16*n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def define_architecture(self, in_shape, hp_params): # print('in_shape') # print(in_shape) # Verify n_class = 1 # Binary classification print(hp_params) if hp_params == None: p = self.default_space() else: p = hp_params routings = p['routings'] conv01_filters = p['conv01_filters'] conv01_ksize = p['conv01_ksize'] conv02_ksize = p['conv02_ksize'] dim_capsule1 = p['dim_capsule1'] dim_capsule2 = p['dim_capsule2'] n_channels = p['n_channels'] in_shape = in_shape[1:] maxlen = in_shape[0] # Input in_layer = Input(shape=(in_shape[0], in_shape[1], 1), name='input_layer') conv1 = Conv2D(filters=conv01_filters, kernel_size=(4, conv01_ksize), strides=1, padding='valid', activation='relu', name='conv1')(in_layer) conv1 = AveragePooling2D((1, 3))(conv1) primarycaps = PrimaryCap(conv1, dim_capsule=dim_capsule1, n_channels=n_channels, kernel_size=(1, conv02_ksize), strides=2, padding='valid') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_capsule2, routings=routings, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = Input(shape=(n_class, ), name='input_decoder') masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training # masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction x_recon = layers.Dense(512, activation='relu', name='decode_dense_01')(masked_by_y) x_recon = layers.Dense(1024, activation='relu', name='decode_dense_02')(x_recon) # # x_recon = layers.Dropout(.2, name='decode_drop_01')(x_recon) x_recon = layers.Dense(np.prod(in_shape), activation='sigmoid', name='decode_dense_03')(x_recon) x_recon = layers.Reshape(target_shape=in_shape, name='out_recon')(x_recon) # Shared Decoder model in training and prediction # decoder = Sequential(name='decoder') # decoder.add(Dense(512, activation='relu', input_dim=16*n_class)) # decoder.add(Dense(1024, activation='relu')) # x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon) # decoder.add(Dense(np.prod((in_shape[1], in_shape[2],1)), activation='sigmoid')) # decoder.add(Reshape(target_shape=(in_shape[1], in_shape[2],1), name='out_recon')) # train_model = Model([in_layer, y], [out_caps, decoder(masked_by_y)]) train_model = Model([in_layer, y], [out_caps, x_recon]) return train_model
def capsModel(input_shape, n_class, num_routing): x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=512, kernel_size=7, strides=1, padding='valid', activation='relu', name='conv1')(x) BN1 = layers.BatchNormalization()(conv1) conv2 = layers.Conv2D(filters=256, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv2')(BN1) BN2 = layers.BatchNormalization()(conv2) conv3 = layers.Conv2D(filters=256, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv3')(BN2) BN3 = layers.BatchNormalization()(conv3) conv4 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv4')(BN3) BN4 = layers.BatchNormalization()(conv4) conv5 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv5')(BN4) BN5 = layers.BatchNormalization()(conv5) conv6 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv6')(BN5) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv6, dim_capsule=24, n_channels=32, kernel_size=12, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def MultiLevelDCNet(input_shape, n_class, routings): """ A Multi-level DCNet on CIFAR-10. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. """ x = layers.Input(shape=input_shape) concat_axis = 1 if K.image_data_format() == 'channels_first' else -1 ########################### Level 1 Capsules ########################### # Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate. conv, nb_filter = densenet.DenseBlock(x, growth_rate=32, nb_layers=8, nb_filter=32) # Batch Normalization DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv) # Creating Primary Capsules (Level 1) # Here PrimaryCapsConv2D is the Conv2D output which is used as the primary capsules by reshaping and squashing (squash activation). # primarycaps_1 (size: [None, num_capsule, dim_capsule]) is the "reshaped and sqashed output" which will be further passed to the dynamic routing protocol. primarycaps_1, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid') # Applying ReLU Activation to primary capsules conv = layers.Activation('relu')(PrimaryCapsConv2D) ########################### Level 2 Capsules ########################### # Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate. conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32) # Batch Normalization DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv) # Creating Primary Capsules (Level 2) primarycaps_2, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid') # Applying ReLU Activation to primary capsules conv = layers.Activation('relu')(PrimaryCapsConv2D) ########################### Level 3 Capsules ########################### # Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate. conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32) # Batch Normalization DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv) # Creating Primary Capsules (Level 3) primarycaps_3, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=3, strides=2, padding='valid') # Merging Primary Capsules for the Merged DigitCaps (CapsuleLayer formed by combining all levels of primary capsules) mergedLayer = layers.merge([primarycaps_1,primarycaps_2,primarycaps_3], mode='concat', concat_axis=1) ########################### Separate DigitCaps Outputs (used for training) ########################### # Merged DigitCaps digitcaps_0 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps0')(mergedLayer) out_caps_0 = Length(name='capsnet_0')(digitcaps_0) # First Level DigitCaps digitcaps_1 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps1')(primarycaps_1) out_caps_1 = Length(name='capsnet_1')(digitcaps_1) # Second Level DigitCaps digitcaps_2 = CapsuleLayer(num_capsule=n_class, dim_capsule=12, routings=routings, name='digitcaps2')(primarycaps_2) out_caps_2 = Length(name='capsnet_2')(digitcaps_2) # Third Level DigitCaps digitcaps_3 = CapsuleLayer(num_capsule=n_class, dim_capsule=10, routings=routings, name='digitcaps3')(primarycaps_3) out_caps_3 = Length(name='capsnet_3')(digitcaps_3) ########################### Combined DigitCaps Output (used for evaluation) ########################### digitcaps = layers.merge([digitcaps_1,digitcaps_2,digitcaps_3, digitcaps_0], mode='concat', concat_axis=2, name='digitcaps') out_caps = Length(name='capsnet')(digitcaps) # Reconstruction (decoder) network y = layers.Input(shape=(n_class,)) masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(600, activation='relu', input_dim=int(digitcaps.shape[2]*n_class), name='zero_layer')) decoder.add(layers.Dense(600, activation='relu', name='one_layer')) decoderFinal = models.Sequential(name='decoderFinal') # Concatenating two layers decoderFinal.add(layers.Merge([decoder.get_layer('zero_layer'), decoder.get_layer('one_layer')], mode='concat')) decoderFinal.add(layers.Dense(1200, activation='relu')) decoderFinal.add(layers.Dense(np.prod([32,32,1]), activation='sigmoid')) decoderFinal.add(layers.Reshape(target_shape=[32,32,1], name='out_recon')) # Model for training train_model = models.Model([x, y], [out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked_by_y)]) # Model for evaluation (prediction) # Note that out_caps is the final prediction. Other predictions could be used for analysing separate-level predictions. eval_model = models.Model(x, [out_caps, out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked)]) return train_model, eval_model
def CapsNet(gene, input_shape, n_class, routings): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ inputs = layers.Input(shape=input_shape) x = inputs print("### Input shape", ) # if input_shape==(28, 28, 1): # dataset="mnist" # elif input_shape==(32, 32, 3): # dataset="cifar" # elif input_shape==(32, 32, 3) and gene[len(gene)-1][0]==2: # dataset="cifar_resized" remaining=len(gene)-2 conv_index=1 caps_index=1 deepcaps_index=1 count=0 xtra_skip=gene[len(gene)-2][0] print("xtra_skip is " +str(gene[len(gene)-2][0])+"\n") # Scan for type of layers #N_layers = int((len(gene)-2)/9) # -1 for xtraskip term conv=[] caps=[] dcaps=[] N_conv=0 N_caps=0 N_dcaps=0 i=0 prevlayertype=0 for index in range(0,len(gene)): if gene[index][0]==0: N_conv = N_conv+1 conv.append(i) i+=1 elif gene[index][0]==1: type=1 N_caps = N_caps+1 caps.append(i) i+=1 elif gene[index][0]==2: convert=1 type=2 N_dcaps = N_dcaps+1 dcaps.append(i) i+=1 for index in range(0,len(gene)): #for layer in gene: if len(gene[index])>1: # Convolutional layers # conv = [0, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules] if gene[index][0]==0: if prevlayertype==2: x=layers.Reshape((gene[index][1], gene[index][1], -1))(x) print("ConvertToConv"+str(prevlayertype)) x = layers.Conv2D(filters=gene[index][7]*gene[index][8], kernel_size=gene[index][4], strides=gene[index][5], padding='same', data_format="channels_last")(x) x = BatchNormalization()(x) print(x.get_shape().as_list()) conv_index=conv_index+1 remaining=remaining-1 print("Added Conv%s_layer" % str(conv_index-1)) count +=1 if xtra_skip==count-1 or (xtra_skip==0 and count==1): x1 = x xtra_skip=-2 #flag==2 for inserted xtra skip prevlayertype=0 # Primary Capsules layers # caps = [1, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules] elif gene[index][0]==1 and remaining>1: x = PrimaryCap(x, dim_capsule=gene[index][8], n_channels=gene[index][7], kernel_size=gene[index][4], strides=gene[index][5], padding='same') #CapsuleLayer(layer[7], layer[8])(x) print(x.get_shape().as_list()) caps_index=caps_index+1 remaining=remaining-1 print("rem "+str(remaining)) print("Added Caps%s_layer" % str(caps_index-1)) veclen=gene[index][7] numout=gene[index][8] prevlayertype=1 # Class Capsules layer # caps = [1, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules] elif gene[index][0]==1 and remaining==1: print(x.get_shape().as_list()) x = layers.Reshape(target_shape=[-1, gene[index][2]])(x) #[veclen*layer[1]*layer[1], numout])(x) # added this to solve error instead of Reshape(target_shape=[-1, layer[2]])(x) print(x.get_shape().as_list()) x = CapsuleLayer(gene[index][7], gene[index][8], 3, name='digitcaps')(x) print(x.get_shape().as_list()) dim_capsule_out=gene[index][8] caps_index=caps_index+1 remaining=remaining-1 print(x.get_shape().as_list()) print("Added ClassCaps_layer\n") # DeepCaps Cells # d_caps = [2, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules] elif gene[index][0]==2 and remaining>1: if prevlayertype==0: x = ConvertToCaps()(x) print("ConvertToCaps"+str(prevlayertype)) print(x.get_shape().as_list()) x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(gene[index][5], gene[index][5]), r_num=1, b_alphas=[1, 1, 1])(x) deepcaps_index=deepcaps_index+1 if remaining==2: x_skip = ConvCapsuleLayer3D(kernel_size=3, num_capsule=gene[index][7], num_atoms=gene[index][8], strides=1, padding='same', routings=3)(x) deepcaps_index=deepcaps_index+1 else: x_skip = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x) deepcaps_index=deepcaps_index+1 x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x) deepcaps_index=deepcaps_index+1 x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x) x = layers.Add()([x, x_skip]) count +=1 #CapsCell count print("Added CapsCell_layer\n") if xtra_skip-1==count-1: x1 = x xtra_skip=-2 #flag==2 for inserted xtra skip elif remaining==1 and xtra_skip==-2: x2 = x elif remaining==2 and xtra_skip==-1: x1 = x x2 = x deepcaps_index=deepcaps_index+1 remaining=remaining-1 x2 = x prevlayertype=2 #FlattenCaps elif gene[index][0]==2 and remaining==1: print(x1.get_shape().as_list()) print(x2.get_shape().as_list()) flatCaps = FlattenCaps() xa = flatCaps(x2) flatCaps = FlattenCaps() xb = flatCaps(x1) x = layers.Concatenate(axis=-2)([xa, xb]) print(xa.get_shape().as_list()) x = DClassCaps(num_capsule=gene[index][7], dim_capsule=gene[index][8], routings=3, channels=0, name='digit_caps')(x) print("Added FlattenCaps_layer\n") print(x.get_shape().as_list()) dim_capsule_out=gene[index][8] else: break digitcaps = x # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class,)) masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') # DeepCaps Decoder if input_shape == (32, 32, 3): # cifar10 resized decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16 decoder.add(layers.Reshape((8, 8, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) #decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(3, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(32, 32, 3), name='out_recon')) # 64, 64 in origial code elif input_shape == (64, 64, 3): # cifar 10 resized decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16 decoder.add(layers.Reshape((8, 8, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(3, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(64, 64, 3), name='out_recon')) elif input_shape == (28, 28, 1): # mnist decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16 decoder.add(layers.Reshape((7, 7, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(1, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(28, 28, 1), name='out_recon')) elif input_shape == (56, 56, 1): # mnist resized decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16 decoder.add(layers.Reshape((7, 7, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(4, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(56, 56, 1), name='out_recon')) elif input_shape == (56, 56, 3): # mnist resized decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16 decoder.add(layers.Reshape((7, 7, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(12, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.summary() decoder.add(layers.Reshape(target_shape=(56, 56, 3), name='out_recon')) elif input_shape == (64, 64, 1): # mnist resized for deepcaps (64x64 inputs) decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16 decoder.add(layers.Reshape((8, 8, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(1, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(64, 64, 1), name='out_recon')) else: raise NotImplementedError(f"Unknown decoder for shape {input_shape}") decoder.summary() # Models for training and evaluation (prediction) train_model = models.Model([inputs, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(inputs, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, dim_capsule_out)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([inputs, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, num_routing): from keras import layers, models from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask from keras.preprocessing import sequence """ A Capsule Network on MNIST. :param input_shape: data shape, 4d, [None, width, height, channels] :param n_class: number of classes :param num_routing: number of routing iterations :return: A Keras Model with 2 inputs and 2 outputs """ x = layers.Input(shape=(maxlen, 1), dtype='float32') # pool = layers.MaxPooling1D(pool_size=3, strides=1)(x) # conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(embed) conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) conv1 = layers.Dropout(0.1)(conv1) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector] # primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. # digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps) digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='out_caps')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def build_the_graph(self, params): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ self.import_nn_parameters(params) start = time() x = layers.Input(shape=self.input_shape) # Layer 1: Just a conventional Conv2D layer # params_conv_layer = self.params[0] conv1 = layers.Conv2D(filters=self.conv_layer['filters'], kernel_size=self.conv_layer['kernel_size'], strides=self.conv_layer['strides'], padding=self.conv_layer['padding'], activation=self.conv_layer['activation'], name=self.conv_layer['activation'])(x) # filters=128, # kernel_size=9, # strides=1, # padding='valid', # activation='relu', # name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap( conv1, dim_capsule=self.primary_caps_layer['dim_capsule'], n_channels=self.primary_caps_layer['n_channels'], kernel_size=self.primary_caps_layer['kernel_size'], strides=self.primary_caps_layer['strides'], padding=self.primary_caps_layer['padding']) # dim_capsule=8, # n_channels=32, # kernel_size=2, # strides=2, # padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer( num_capsule=self.n_class, dim_capsule=self.digit_caps_layer['dim_capsule'], # /dim_capsule = 16 routings=self.routings, name=self.digit_caps_layer['name'])(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(self.n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask()( digitcaps ) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add( layers.Dense(self.decoder_layer['first_dense'], activation='relu', input_dim=self.digit_caps_layer['dim_capsule'] * self.n_class)) decoder.add( layers.Dense(self.decoder_layer['second_dense'], activation='relu')) # decoder.add(layers.Dropout(0.5)) # decoder.add(layers.Dense(128, activation='relu', input_dim=16 * self.n_class)) # decoder.add(layers.Dense(256, activation='relu')) # decoder.add(layers.Dense(np.prod(self.input_shape), activation='sigmoid')) decoder.add( layers.Dense(np.prod(self.input_shape), activation='softmax')) decoder.add( layers.Reshape(target_shape=self.input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input( shape=(self.n_class, self.digit_caps_layer['dim_capsule'])) # 16 noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) self.train_model = train_model self.eval_model = eval_model self.manipulate_model = manipulate_model print('time to generate the model: {}'.format(time() - start)) return train_model, eval_model, manipulate_model
from keras import layers, models, optimizers from keras.preprocessing.image import ImageDataGenerator from keras import backend as K n_class=2 def CapsNet(input_shape, n_class, routings): x = layers.Input(shape=input_shape) conv1 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv1')(x) primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=3, kernel_size=9, strides=2, padding='valid') CTscnaCaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,name='CTscnaCaps')(primarycaps) out_caps = Length(name='capsnet')(CTscnaCaps) y = layers.Input(shape=(n_class,)) masked_by_y = Mask()([CTscnaCaps, y]) masked = Mask()(CTscnaCaps) decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16*n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) noise = layers.Input(shape=(n_class, 16)) noised_CTscnaCaps = layers.Add()([CTscnaCaps, noise]) masked_noised_y = Mask()([noised_CTscnaCaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=num_classes, dim_capsule=16, num_routing=3, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(num_classes, )) masked_by_y = Mask()([ digitcaps, y ]) # The true label is used to mask the output of capsule layer. For training masked = Mask()( digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * num_classes)) decoder.add(layers.Dropout(0.2)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)])
def CapsNet(input_shape, n_class, routings): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv0 = layers.Conv2D(filters=64, kernel_size=5, strides=1, padding='valid', activation='relu', name='conv0')(x) #conv1 = layers.Conv2D(filters=128, kernel_size=5, strides=1, padding='valid', activation='relu', name='conv1')(conv0) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv0, dim_capsule=8, n_channels=32, kernel_size=5, strides=1, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction ''' decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(256, activation='linear', input_dim=16*n_class)) decoder.add(layers.Dense(512, activation='linear')) decoder.add(layers.Dense(1024, activation='linear')) decoder.add(layers.Dense(np.prod(input_shape), activation='linear')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) decoder.summary() ''' decoder = models.Sequential(name='decoder') decoder.add( layers.Dense(8 * 8 * 4, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Reshape((8, 8, 4))) decoder.add( layers.Conv2D(128, kernel_size=5, strides=1, padding='same', activation='relu')) decoder.add(layers.UpSampling2D()) #16x16x128 decoder.add( layers.Conv2D(64, kernel_size=5, strides=1, padding='same', activation='relu')) decoder.add(layers.UpSampling2D()) #32x32x64 decoder.add( layers.Conv2D(3, kernel_size=5, strides=1, padding='same', activation='tanh')) decoder.summary() #64x64x3 # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def MultiLevelDCNet(input_shape, n_class, routings): """ A DCNet (1-level DCNet) on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. """ x = layers.Input(shape=input_shape) concat_axis = 1 if K.image_data_format() == 'channels_first' else -1 ########################### Primary Capsules ########################### # Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate. conv, nb_filter = densenet.DenseBlock(x, growth_rate=32, nb_layers=8, nb_filter=32) # Batch Normalization DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv) # Creating Primary Capsules # Here PrimaryCapsConv2D is the Conv2D output which is used as the primary capsules by reshaping and squashing (squash activation). # primarycaps_1 (size: [None, num_capsule, dim_capsule]) is the "reshaped and sqashed output" which will be further passed to the dynamic routing protocol. primarycaps, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') ########################### DigitCaps Output ########################### digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps0')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) # Reconstruction (decoder) network y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add( layers.Dense(512, activation='relu', input_dim=int(digitcaps.shape[2] * n_class), name='zero_layer')) decoder.add(layers.Dense(512, activation='relu', name='one_layer')) decoderFinal = models.Sequential(name='decoderFinal') # Concatenating two layers decoderFinal.add( layers.Merge( [decoder.get_layer('zero_layer'), decoder.get_layer('one_layer')], mode='concat')) decoderFinal.add(layers.Dense(1024, activation='relu')) decoderFinal.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoderFinal.add(layers.Reshape(input_shape, name='out_recon')) # Model for training train_model = models.Model([x, y], [out_caps, decoderFinal(masked_by_y)]) # Model for evaluation (prediction) eval_model = models.Model(x, [out_caps, decoderFinal(masked)]) return train_model, eval_model
def CapsNet_WithDecoder(input_shape, n_class, kernel, primary_channel, primary_veclen, digit_veclen, dropout, routings, decoderParm): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ if kernel > 19: conv_padding = 'same' print('kernel size big, padding to same') else: conv_padding = 'valid' x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=kernel, strides=1, padding=conv_padding, activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=primary_veclen, n_channels=primary_channel, kernel_size=kernel, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=digit_veclen, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction NumDecoderLayer, DecoderLayerUnit = decoderParm decoder = models.Sequential(name='decoder') decoder.add( layers.Dense(DecoderLayerUnit[0], activation='relu', input_dim=digit_veclen * n_class)) #16*n_class)) for i in range(NumDecoderLayer - 2): decoder.add(layers.Dense(DecoderLayerUnit[i + 1], activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, digit_veclen)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings): """ A Capsule Network on CIFAR-10. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D( filters=256, kernel_size=9, strides=1, padding="valid", activation="relu", name="conv1", )(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to # [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=64, kernel_size=9, strides=2, padding="valid") # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name="digitcaps")(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with # its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name="capsnet")(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name="decoder") decoder.add(layers.Dense(512, activation="relu", input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation="relu")) decoder.add(layers.Dense(np.prod(input_shape), activation="sigmoid")) decoder.add(layers.Reshape(target_shape=input_shape, name="out_recon")) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings): """ MNISTに関するカプセルネットワーク :param input_shape: 入力データのshape(3次元で[width, height, channels]という形) :param n_class: クラスの数 :param routings: routingを行う回数 :return 2つのmodel (1つ目:学習用モデル, 2つ目:評価用モデル) `eval_model`というモデルも学習用としてしようすることもできる. """ x = layers.Input(shape=input_shape) # 1層目: ただの2次元畳み込み層 conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # 2層目: 活性化関数にsquash関数を用いた2次元畳み込み層で,[None ,num_capsule, dim_capsule]という形に変換する primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # 3層目: カプセル層 (routingアルゴリズムはここで行っている) digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # 4層目: ここはカプセルを"長さ"に変形するための補助レイヤーで, 教師データの形に合わせている. # tensorflowを使用している場合, ここは必要ありません. out_caps = Length(name='capsnet')(digitcaps) # Decoderネットワーク y = layers.Input(shape=(n_class, )) masked_by_y = Mask()([digitcaps, y]) # 正解のラベルはよく学習のためにカプセル層の出力を隠す masked = Mask()(digitcaps) # マスクは予測のために長さが最大値のカプセルを使用する # このDecoderモデルは学習時にも予測時にも使われる. decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # 学習モデルと評価モデル train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # モデルにノイズを加える noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ print(input_shape) print(n_class) input_shape = (config.MAX_TEXT_LENGTH, ) n_class = 6 print(input_shape) print(n_class) x = layers.Input(shape=(config.MAX_TEXT_LENGTH, )) x1 = layers.Embedding( config.MAX_FEATURES, config.embedding_dims, input_length=config.MAX_TEXT_LENGTH, # weights=[embedding_matrix], trainable=True)(x) x1 = layers.Dropout(0.2)(x1) x1 = Reshape((config.MAX_TEXT_LENGTH, -1, 1))(x1) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x1) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # lr=layers.Dense(n_class, activation="sigmoid", name='softmax')(digitcaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # out_caps = layers.Dense(n_class, activation="sigmoid", name='softmax')(out_caps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) # train_model.summary() # eval_model.summary() # manipulate_model.summary() return train_model, eval_model, manipulate_model