def CapsNet(input_shape, n_class, routings): x = layers.Input(shape=input_shape) embed = layers.Embedding(output_dim=emb_output_dim, input_dim=length, trainable=emb_train)(x) conv1 = layers.Conv1D(filters=conv1_filt, kernel_size=conv1_kern, strides=conv1_stride, padding=conv1_pad, activation=conv1_act, name='conv1')(embed) primarycaps = PrimaryCap(conv1, dim_capsule=caps1_dim, n_channels=caps1_n_channels, kernel_size=caps1_kern, strides=caps1_stride, padding=caps1_pad) digitcaps = CapsuleLayer(num_capsule=caps2_num, dim_capsule=caps2_dim, routings=routings, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) train_model = models.Model(x, out_caps) return train_model
def build_discriminator(): img = Input(shape=img_shape) # x = UpSampling2D()(x) # x = Conv2D(128, kernel_size=3, strides=1, padding="same")(img) # x = LeakyReLU(alpha=0.2)(x) # x = BatchNormalization(momentum=0.8)(x) x = Conv2D(256, kernel_size=9, strides=1, padding="valid")(img) x = LeakyReLU(alpha=0.2)(x) x = BatchNormalization(momentum=0.8)(x) x = PrimaryCap(x, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') x = CapsuleLayer(num_capsule=10, dim_capsule=32, routings=routings)(x) # x = Mask()(x) # y = layers.Input(shape=(num_classes+1,)) # x2 = Mask()([x, y]) # # x2 = Length(name='capsnet')(x) # x1 = Mask()(x) x = Flatten()(x) # Determine validity and label of the image validity = Dense(1, activation="sigmoid", name='ds')(x) label = Dense(num_classes, activation="softmax", name='label')(x) return Model(img, [validity, label])
def CapsNet(input_shape, n_class, routings): x = Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = Conv2D(filters=128, kernel_size=9, strides=2, padding='valid', activation='relu', name='conv1')(x) primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=16, kernel_size=9, strides=4, padding='valid') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) out_caps = Flatten()(digitcaps) # outputs = Dense(278)(out_caps) # Models for training and evaluation (prediction) train_model = models.Model(x, out_caps) return train_model
def CapsNet(input_shape, n_class, num_routing, kernel_size=7): """ A Capsule Network on 2019ncon. :param input_shape: data shape, 3d, [width, height, channels],for example:[21,28,1] :param n_class: number of classes :param num_routing: number of routing iterations :return: A Keras Model with 2 inputs and 2 outputs """ x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=128, kernel_size=kernel_size, strides=1, padding='same', activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector] primarycaps = PrimaryCap(conv1, dim_vector=16, n_channels=32, kernel_size=kernel_size, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=32, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='out_caps')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class,)) masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. x_recon = layers.Dense(512, activation='relu')(masked) x_recon = layers.Dense(1024, activation='relu')(x_recon) x_recon = layers.Dense(np.prod(input_shape), activation='sigmoid')(x_recon) x_recon = layers.Reshape(target_shape=input_shape, name='out_recon')(x_recon) # two-input-two-output keras Model return models.Model([x, y], [out_caps, x_recon])
def CNN(seq_length, length, input_size, feature_maps, kernels, x): concat_input = [] for feature_map, kernel in zip(feature_maps, kernels): reduced_l = length - kernel + 1 #changes # conv1 = Conv2D(feature_map, (1, kernel), activation='tanh', data_format="channels_last")(x) conv1 = Conv2D(feature_map, (1, kernel), strides=1, activation='tanh', data_format="channels_last")(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector] primarycaps = PrimaryCap(conv1, dim_vector=feature_map, n_channels=32, kernel_size=(1, kernel), strides=2, padding='valid', filters=feature_map) # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=seq_length, dim_vector=feature_map, num_routing=3)(primarycaps) # conv = Conv2D(feature_map, (1, kernel), activation='tanh', data_format="channels_last")(x) # maxp = MaxPooling2D((1, reduced_l), data_format="channels_last")(conv) concat_input.append(digitcaps) x = Concatenate()(concat_input) x = Reshape((seq_length, sum(feature_maps)))(x) return x
def CapsNet(input_shape, n_class, num_routing): x = layers.Input(shape=input_shape) conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu')(x) primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing)(primarycaps) out_caps = Length(name='out_caps')(digitcaps) # Reconstruction network y = layers.Input(shape=(n_class, )) masked = Mask()([digitcaps, y]) x_recon = layers.Dense(512, activation='relu')(masked) x_recon = layers.Dense(1024, activation='relu')(x_recon) x_recon = layers.Dense(784, activation='sigmoid')(x_recon) x_recon = layers.Reshape(target_shape=[28, 28, 1], name='out_recon')(x_recon) return models.Model([x, y], [out_caps, x_recon])
def prepareModel(): """ Capsule Network on biopsy patches. """ num_routing = 3 capsule_dimension = 8, no_of_channels = 16 x = Input(shape=(224, 224, 3)) # Layer 1: Just a conventional Conv2D layer conv1 = Conv2D(filters=32, kernel_size=5, strides=2, padding='valid', name='conv1')(x) conv1 = Activation('relu', name='relu1')(conv1) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=16, kernel_size=5, strides=2, padding='valid') primarycaps = Reshape((53 * 53, 16, 8))(primarycaps) # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = TimeDistributed( CapsuleLayer(num_capsule=1, dim_capsule=8, num_routing=num_routing, name='digitcaps'))(primarycaps) out_caps = Length(name='capsnet')(digitcaps) output = TrimmedAveragePool()(out_caps) model = Model(inputs=x, outputs=output) model.summary() return model
def CapsNet(input_shape, n_class, num_routing): """ A Capsule Network on MNIST. :param input_shape: data shape, 4d, [None, width, height, channels] :param n_class: number of classes :param num_routing: number of routing iterations :return: A Keras Model with 2 inputs and 2 outputs """ x = layers.Input(shape=(maxlen,)) embed = layers.Embedding(max_features, embed_dim, input_length=maxlen)(x) conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')( embed) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector] primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='out_caps')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class,)) masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. x_recon = layers.Dense(512, activation='relu')(masked) x_recon = layers.Dense(1024, activation='relu')(x_recon) x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon) # two-input-two-output keras Model return models.Model([x, y], [out_caps, x_recon])
def make_model(): x = Input(shape=input_shape) conv1 = Conv3D(filters=conv_layer_filters, kernel_size=first_layer_kernel_size, strides=1, padding='valid', activation='relu', name='conv1')(x) primarycaps = PrimaryCap(conv1, dim_capsule=dim_primary_capsule, n_channels=n_channels, kernel_size=primary_cap_kernel_size, strides=2, padding='valid', name='primarycap_conv3d') sub_caps = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_sub_capsule, routings=3, name='sub_caps')(primarycaps) out_caps = Length(name='capsnet')(sub_caps) # Decoder network y = Input(shape=(n_class,)) masked_by_y = Mask()([sub_caps, y]) masked = Mask()(sub_caps) # shared decoder model in training and prediction decoder = Sequential(name='decoder') decoder.add(Dense(512, activation='relu', input_dim=dim_sub_capsule*n_class)) decoder.add(Dense(1024, activation='relu')) decoder.add(Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(Reshape(target_shape=input_shape, name='out_recon')) ### Models for training and evaluation (prediction and actually using) train_model = Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = Model(x, [out_caps, decoder(masked)]) ### manipulate model can be used to visualize activation maps for specific classes noise = Input(shape=(n_class, dim_sub_capsule)) noised_sub_caps = Add()([sub_caps, noise]) masked_noised_y = Mask()([noised_sub_caps, y]) manipulate_model = Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings, model_version, lam_regularize): """ A Capsule Network . :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer, 对DEAP,kernel_size=9;对DREAMER,kernel_size=6 conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1', kernel_regularizer=regularizers.l2(lam_regularize))( x) #kernel_size=9 # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] # 对DEAP,kernel_size=9;对DREAMER,kernel_size=6 # 对CapsNet,strides=2,pading=‘valid’;对MLF-CapsNet,stides=1,padding='same' if model_version == 'v0': primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid', lam_regularize=lam_regularize, model_version=model_version) else: primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=1, padding='same', lam_regularize=lam_regularize, model_version=model_version) #kernel_size=9 # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps', lam_regularize=lam_regularize)(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Models for training and evaluation (prediction) y = layers.Input(shape=(n_class, )) train_model = models.Model([x, y], out_caps) eval_model = models.Model(x, out_caps) return train_model, eval_model
def CapsNet(input_shape, n_class, routings, batch_size): x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) out = models.Model(inputs=x, outputs=out_caps) return out
def CapsNet(input_shape, n_class, num_routing): """ MNIST Veriseti için Kapsül Ağları. :param input_shape: veri şekli, 3d, [genişlik, yükseklik, kanal] :param n_class: sınıf sayısı :param num_routing: dinamik yönlendirme (dynamic routing) iterasyon sayısı :return: iki Keras model, birincisi eğitim için, ikincisi değerlendirme için (evalaution). `eval_model` aynı zamanda eğitim için de kullanılabilir. """ x = layers.Input(shape=input_shape) # Katman 1: Geleneksel Evrişimli Sinir Ağı Katmanı (Conv2D) conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Katman 2: Conv2D katmanı ile ezme (squash) aktivasyonu, [None, num_capsule, dim_capsule]’e yeniden şekil veriliyor. primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Katman 3: Kapsül Katmanı. Dinamik Yönlendirme algoritması burada çalışıyor. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Katman 4: Her kapsülün uzunluğunu yeniden düzenleyen yardımcı bir katmandır. Doğru etiketle eşleşmesi için bu işlem yapılır. # Eğer Tensorflow kullanıyorsanız bu işleme gerek yoktur. :) out_caps = Length(name='capsnet')(digitcaps) # Kodçözücü Ağ. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # Doğru etiket, kapsül katmanın çıkışını maskelemek için kullanılır (Eğitim için). masked = Mask()( digitcaps ) # Filtre (maske), kapsülün maksimal uzunluğu ile kullanılır (Kestirim için). # Eğitim ve Kestirimde Kodçözücü Modelin Paylaşımı decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Eğitim ve Değerlendirme (Kestirim) için Modeller train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet(input_shape, n_class, routings): x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) # YES decoder.add(layers.Dense(1024, activation='relu')) # YES decoder.add(layers.Dense(4096, activation='relu')) # YES decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, num_routing): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param num_routing: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction other_masks = [] # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet(input_shape, n_class, routings): x = layers.Input(shape=input_shape) # Layers 1: Convolution Layers to extract low level features conv1 = layers.Conv2D(filters=64, kernel_size=3, strides=2, activation='relu', name='Conv1')(x) conv2 = layers.Conv2D(filters=128, kernel_size=3, strides=2, activation='relu', name='Conv2')(conv1) conv3 = layers.Conv2D(filters=256, kernel_size=6, strides=2, activation='relu', name='Conv3')(conv2) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv3, dim_capsule=8, n_channels=32, kernel_size=6, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. classcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='classcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. out_caps = Length(name='capsnet')(classcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()([classcaps, y]) masked = Mask()(classcaps) # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet(input_shape, n_class, routings): """ 针对 MNIST 手写字符识别的胶囊网络 ## input_shape: 输入数据的维度, 长度为3的列表, [width, height, channels] ## n_class: 类别的数量 ## routings: routing算法迭代的次数 :return: 返回两个模型, 第一个用于训练, 第二个用于测试 """ # Encoder network.输入为图片 x = layers.Input(shape=input_shape) # Layer 1: 卷积层 conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # Layer 2: PrimaryCap(自定义层) primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: CapsuleLayer(自定义胶囊层) digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # Layer 4: 模值计算层(自定义层) out_caps = Length(name='capsnet')(digitcaps) # Decoder network.输入为向量 y = layers.Input(shape=(n_class, )) # 蒙版操作,对decoder输入的标准化 masked_by_y = Mask()([digitcaps, y]) # 用真实值取代胶囊层的输出值。此项用于训练 masked = Mask()(digitcaps) # 用最大长度值做胶囊的蒙版。此项用于评估 # 包含3层的全连接神经网络(序贯模型) decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # 最终将输出转为图片 # 训练模型和评估模型 train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def build(self): query = Input(name='query', shape=(self.config['text1_maxlen'],)) show_layer_info('Input', query) doc = Input(name='doc', shape=(self.config['text2_maxlen'],)) show_layer_info('Input', doc) dpool_index = Input(name='dpool_index', shape=[self.config['text1_maxlen'], self.config['text2_maxlen'], 3], dtype='int32') show_layer_info('Input', dpool_index) embedding = Embedding(self.config['vocab_size'], self.config['embed_size'], weights=[self.config['embed']], trainable = self.embed_trainable) q_embed = embedding(query) show_layer_info('Embedding', q_embed) d_embed = embedding(doc) show_layer_info('Embedding', d_embed) cross = Dot(axes=[2, 2], normalize=False)([q_embed, d_embed]) show_layer_info('Dot', cross) cross_reshape = Reshape((self.config['text1_maxlen'], self.config['text2_maxlen'], 1))(cross) show_layer_info('Reshape', cross_reshape) conv2d = Conv2D(self.config['kernel_count'], self.config['kernel_size'], padding='same', activation='relu') dpool = DynamicMaxPooling(self.config['dpool_size'][0], self.config['dpool_size'][1]) conv1 = conv2d(cross_reshape) show_layer_info('Conv2D', conv1) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=2, dim_capsule=16, routings=3, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) pool1 = dpool([conv1, dpool_index]) show_layer_info('DynamicMaxPooling', pool1) pool1_flat = Flatten()(pool1) show_layer_info('Flatten', pool1_flat) pool1_flat_drop = Dropout(rate=self.config['dropout_rate'])(pool1_flat) show_layer_info('Dropout', pool1_flat_drop) if self.config['target_mode'] == 'classification': out_ = Dense(2, activation='softmax')(pool1_flat_drop) elif self.config['target_mode'] in ['regression', 'ranking']: out_ = Dense(1)(pool1_flat_drop) show_layer_info('Dense', out_) model = Model(inputs=[query, doc, dpool_index], outputs=out_) model.summary() return model
def CapsNet_NoDecoder(input_shape, n_class, kernel, primary_channel, primary_veclen, digit_veclen, dropout, routings, model_size_info): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. model_size_info: kernel, conv_channel, primary_channel, input_veclen, output_veclen """ kernel, conv_channel, primary_channel, primary_veclen, digit_veclen = model_size_info if kernel > 19: conv_padding = 'same' print('kernel size big, padding to same') else: conv_padding = 'valid' x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=kernel, strides=1, padding=conv_padding, activation='relu', name='conv1')(x) if dropout != 0: conv1 = layers.Dropout(dropout)(conv1) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=primary_veclen, n_channels=primary_channel, kernel_size=kernel, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=digit_veclen, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Models for training and evaluation (prediction) train_model = models.Model(x, out_caps) return train_model
def build_model(input_shape): # encoder x = Input(shape=input_shape) conv1 = Conv2D(filters=64, kernel_size=3, padding='same', trainable=True, name='conv1')(x) conv1 = BatchNormalization()(conv1) conv1 = Activation('relu')(conv1) conv1 = Conv2D(filters=64, kernel_size=3, padding='same', trainable=True, name='conv2')(conv1) conv1 = BatchNormalization()(conv1) conv1 = Activation('relu')(conv1) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=N_CLASS, dim_capsule=16, routings=ROUTINGS, name='digitcaps')(primarycaps) digitcaps = Flatten()(digitcaps) decoder = Dense(512, activation='relu', input_dim=16 * N_CLASS)(digitcaps) decoder = Dense(1024, activation='relu')(decoder) decoder = Dense(np.prod(input_shape), activation='sigmoid')(decoder) decoder = Reshape(input_shape)(decoder) model = Model(x, decoder) # transfer weights from first 2 layers of VGG-19 weights = pre_trained_model.layers[1].get_weights()[0][:, :, :, :] #weights = np.reshape(weights, (3,3,1,64)) bias = pre_trained_model.layers[1].get_weights()[1] model.layers[1].set_weights([weights, bias]) weights = pre_trained_model.layers[2].get_weights()[0] bias = pre_trained_model.layers[2].get_weights()[1] model.layers[4].set_weights([weights, bias]) return model
def CapsNet(input_shape, n_class, routings): left_input = layers.Input(shape=input_shape) right_input = layers.Input(shape=input_shape) input = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(input) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) tunnel_l = models.Model(input, out_caps) tunnel_r = models.Model(input, out_caps) encoded_l = tunnel_l(left_input) encoded_r = tunnel_r(right_input) L1_layer = layers.Lambda(lambda tensors: K.abs(tensors[0] - tensors[1])) L1_distance = L1_layer([encoded_l, encoded_r]) prediction = layers.Dense(1, activation='sigmoid')(L1_distance) train_model = models.Model(inputs=[left_input, right_input], outputs=prediction) return train_model
def CapsNet(input_shape, n_class, routings): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D( filters=256, kernel_size=9, strides=4, # TODO: use 5 padding='valid', activation='relu', name='conv1')( x) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap( conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer( num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')( primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Models for training and evaluation (prediction) model = models.Model(x, out_caps) return model
def CapsNet(input_shape, n_class, routings): x = layers.Input(shape=input_shape) conv1 = layers.Conv2D(filters=256, kernel_size=(9), strides=1, padding='valid', activation='relu', name='conv1')(x) primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) y = layers.Input(shape=(n_class, )) masked_by_y = Mask()([digitcaps, y]) masked = Mask()(digitcaps) decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings, batch_size): # input layer x = tf.keras.layers.Input(input_shape, batch_size = batch_size) # layer 1 : regular Convolutionnal layer conv1 = tf.keras.layers.Conv2D(filters = 256, kernel_size = (9,9), activation = 'relu', name = 'conv1')(x) # layer 2 : PrimaryCaps, which is a convolution layer with Squash activation # dim_capsule : corresponds to the dimension of the capsule output vector # n_channels : number of capsule types primarycaps = PrimaryCap(conv1, dim_capsule = 8, n_channels = 32, kernel_size = 9, strides = 2, padding = 'valid') # layer 3 : CapsuleLayer (involves routing by agreement) # each capsule in this layer represents one of the Kuzushiji symbol kcaps = CapsuleLayer(num_capsule = n_class, dim_capsule = 16, routings = routings, name = 'kcaps')(primarycaps) # layer 4 : layer that takes the length of each capsule out_caps = Length(name='capsnet')(kcaps) # Let's build the decoder network # 2 reconstructions are performed : # - first one is to reconstruct image according to the true label # - second one is to reconstruct image according to the vector with maximal length (prediction) y = tf.keras.layers.Input((n_class,)) masked_by_y = Mask()([kcaps, y]) masked = Mask()(kcaps) # Dense layers of the decoder architecture as described in the paper decoder = tf.keras.models.Sequential(name = 'decoder') decoder.add(tf.keras.layers.Dense(512, activation = 'relu', input_dim = 16 * n_class)) decoder.add(tf.keras.layers.Dense(1024, activation = 'relu')) decoder.add(tf.keras.layers.Dense(input_shape[0]*input_shape[1], activation = 'sigmoid')) decoder.add(tf.keras.layers.Reshape(input_shape, name = 'out_recon')) # Models used for training and evaluation # train_model involves training of the decoder # while evaluation model, given an input x, outputs his prediction and his reconstruction using the trained decoder train_model = tf.keras.models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = tf.keras.models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def create_layers(input_img, n_class, routings): # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=128, kernel_size=5, strides=1, padding='valid', activation='relu')(input_img) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=4, routings=routings)(primarycaps) return digitcaps
def create_layers(input_img, n_class, routings): # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=256, kernel_size=5, strides=1, padding='valid', activation='relu')(input_img) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=5, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings)(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) return digitcaps
def CapsNet(input_shape, n_class, routings): """ MNISTに関するカプセルネットワーク :param input_shape: 入力データのshape(3次元で[width, height, channels]という形) :param n_class: クラスの数 :param routings: routingを行う回数 :return 2つのmodel (1つ目:学習用モデル, 2つ目:評価用モデル) `eval_model`というモデルも学習用としてしようすることもできる. """ x = layers.Input(shape=input_shape) # 1層目: ただの2次元畳み込み層 conv1 = layers.Conv2D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(x) # 2層目: 活性化関数にsquash関数を用いた2次元畳み込み層で,[None ,num_capsule, dim_capsule]という形に変換する primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') # 3層目: カプセル層 (routingアルゴリズムはここで行っている) digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps')(primarycaps) # 4層目: ここはカプセルを"長さ"に変形するための補助レイヤーで, 教師データの形に合わせている. # tensorflowを使用している場合, ここは必要ありません. out_caps = Length(name='capsnet')(digitcaps) # Decoderネットワーク y = layers.Input(shape=(n_class, )) masked_by_y = Mask()([digitcaps, y]) # 正解のラベルはよく学習のためにカプセル層の出力を隠す masked = Mask()(digitcaps) # マスクは予測のために長さが最大値のカプセルを使用する # このDecoderモデルは学習時にも予測時にも使われる. decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # 学習モデルと評価モデル train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) # モデルにノイズを加える noise = layers.Input(shape=(n_class, 16)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def MultiLevelDCNet(input_shape, n_class, routings): """ A Multi-level DCNet on CIFAR-10. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. """ x = layers.Input(shape=input_shape) concat_axis = 1 if K.image_data_format() == 'channels_first' else -1 ########################### Level 1 Capsules ########################### # Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate. conv, nb_filter = densenet.DenseBlock(x, growth_rate=32, nb_layers=8, nb_filter=32) # Batch Normalization DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv) # Creating Primary Capsules (Level 1) # Here PrimaryCapsConv2D is the Conv2D output which is used as the primary capsules by reshaping and squashing (squash activation). # primarycaps_1 (size: [None, num_capsule, dim_capsule]) is the "reshaped and sqashed output" which will be further passed to the dynamic routing protocol. primarycaps_1, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid') # Applying ReLU Activation to primary capsules conv = layers.Activation('relu')(PrimaryCapsConv2D) ########################### Level 2 Capsules ########################### # Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate. conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32) # Batch Normalization DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv) # Creating Primary Capsules (Level 2) primarycaps_2, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid') # Applying ReLU Activation to primary capsules conv = layers.Activation('relu')(PrimaryCapsConv2D) ########################### Level 3 Capsules ########################### # Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate. conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32) # Batch Normalization DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv) # Creating Primary Capsules (Level 3) primarycaps_3, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=3, strides=2, padding='valid') # Merging Primary Capsules for the Merged DigitCaps (CapsuleLayer formed by combining all levels of primary capsules) mergedLayer = layers.merge([primarycaps_1,primarycaps_2,primarycaps_3], mode='concat', concat_axis=1) ########################### Separate DigitCaps Outputs (used for training) ########################### # Merged DigitCaps digitcaps_0 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps0')(mergedLayer) out_caps_0 = Length(name='capsnet_0')(digitcaps_0) # First Level DigitCaps digitcaps_1 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, name='digitcaps1')(primarycaps_1) out_caps_1 = Length(name='capsnet_1')(digitcaps_1) # Second Level DigitCaps digitcaps_2 = CapsuleLayer(num_capsule=n_class, dim_capsule=12, routings=routings, name='digitcaps2')(primarycaps_2) out_caps_2 = Length(name='capsnet_2')(digitcaps_2) # Third Level DigitCaps digitcaps_3 = CapsuleLayer(num_capsule=n_class, dim_capsule=10, routings=routings, name='digitcaps3')(primarycaps_3) out_caps_3 = Length(name='capsnet_3')(digitcaps_3) ########################### Combined DigitCaps Output (used for evaluation) ########################### digitcaps = layers.merge([digitcaps_1,digitcaps_2,digitcaps_3, digitcaps_0], mode='concat', concat_axis=2, name='digitcaps') out_caps = Length(name='capsnet')(digitcaps) # Reconstruction (decoder) network y = layers.Input(shape=(n_class,)) masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(600, activation='relu', input_dim=int(digitcaps.shape[2]*n_class), name='zero_layer')) decoder.add(layers.Dense(600, activation='relu', name='one_layer')) decoderFinal = models.Sequential(name='decoderFinal') # Concatenating two layers decoderFinal.add(layers.Merge([decoder.get_layer('zero_layer'), decoder.get_layer('one_layer')], mode='concat')) decoderFinal.add(layers.Dense(1200, activation='relu')) decoderFinal.add(layers.Dense(np.prod([32,32,1]), activation='sigmoid')) decoderFinal.add(layers.Reshape(target_shape=[32,32,1], name='out_recon')) # Model for training train_model = models.Model([x, y], [out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked_by_y)]) # Model for evaluation (prediction) # Note that out_caps is the final prediction. Other predictions could be used for analysing separate-level predictions. eval_model = models.Model(x, [out_caps, out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked)]) return train_model, eval_model
def capsModel(input_shape, n_class, num_routing): x = layers.Input(shape=input_shape) # Layer 1: Just a conventional Conv2D layer conv1 = layers.Conv2D(filters=512, kernel_size=7, strides=1, padding='valid', activation='relu', name='conv1')(x) BN1 = layers.BatchNormalization()(conv1) conv2 = layers.Conv2D(filters=256, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv2')(BN1) BN2 = layers.BatchNormalization()(conv2) conv3 = layers.Conv2D(filters=256, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv3')(BN2) BN3 = layers.BatchNormalization()(conv3) conv4 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv4')(BN3) BN4 = layers.BatchNormalization()(conv4) conv5 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv5')(BN4) BN5 = layers.BatchNormalization()(conv5) conv6 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv6')(BN5) # Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule] primarycaps = PrimaryCap(conv6, dim_capsule=24, n_channels=32, kernel_size=12, strides=2, padding='valid') # Layer 3: Capsule layer. Routing algorithm works here. digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, num_routing=num_routing, name='digitcaps')(primarycaps) # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class, )) masked_by_y = Mask()( [digitcaps, y] ) # The true label is used to mask the output of capsule layer. For training masked = Mask( )(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class)) decoder.add(layers.Dense(1024, activation='relu')) decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid')) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) # Models for training and evaluation (prediction) train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model
def CapsNet(gene, input_shape, n_class, routings): """ A Capsule Network on MNIST. :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. `eval_model` can also be used for training. """ inputs = layers.Input(shape=input_shape) x = inputs print("### Input shape", ) # if input_shape==(28, 28, 1): # dataset="mnist" # elif input_shape==(32, 32, 3): # dataset="cifar" # elif input_shape==(32, 32, 3) and gene[len(gene)-1][0]==2: # dataset="cifar_resized" remaining=len(gene)-2 conv_index=1 caps_index=1 deepcaps_index=1 count=0 xtra_skip=gene[len(gene)-2][0] print("xtra_skip is " +str(gene[len(gene)-2][0])+"\n") # Scan for type of layers #N_layers = int((len(gene)-2)/9) # -1 for xtraskip term conv=[] caps=[] dcaps=[] N_conv=0 N_caps=0 N_dcaps=0 i=0 prevlayertype=0 for index in range(0,len(gene)): if gene[index][0]==0: N_conv = N_conv+1 conv.append(i) i+=1 elif gene[index][0]==1: type=1 N_caps = N_caps+1 caps.append(i) i+=1 elif gene[index][0]==2: convert=1 type=2 N_dcaps = N_dcaps+1 dcaps.append(i) i+=1 for index in range(0,len(gene)): #for layer in gene: if len(gene[index])>1: # Convolutional layers # conv = [0, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules] if gene[index][0]==0: if prevlayertype==2: x=layers.Reshape((gene[index][1], gene[index][1], -1))(x) print("ConvertToConv"+str(prevlayertype)) x = layers.Conv2D(filters=gene[index][7]*gene[index][8], kernel_size=gene[index][4], strides=gene[index][5], padding='same', data_format="channels_last")(x) x = BatchNormalization()(x) print(x.get_shape().as_list()) conv_index=conv_index+1 remaining=remaining-1 print("Added Conv%s_layer" % str(conv_index-1)) count +=1 if xtra_skip==count-1 or (xtra_skip==0 and count==1): x1 = x xtra_skip=-2 #flag==2 for inserted xtra skip prevlayertype=0 # Primary Capsules layers # caps = [1, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules] elif gene[index][0]==1 and remaining>1: x = PrimaryCap(x, dim_capsule=gene[index][8], n_channels=gene[index][7], kernel_size=gene[index][4], strides=gene[index][5], padding='same') #CapsuleLayer(layer[7], layer[8])(x) print(x.get_shape().as_list()) caps_index=caps_index+1 remaining=remaining-1 print("rem "+str(remaining)) print("Added Caps%s_layer" % str(caps_index-1)) veclen=gene[index][7] numout=gene[index][8] prevlayertype=1 # Class Capsules layer # caps = [1, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules] elif gene[index][0]==1 and remaining==1: print(x.get_shape().as_list()) x = layers.Reshape(target_shape=[-1, gene[index][2]])(x) #[veclen*layer[1]*layer[1], numout])(x) # added this to solve error instead of Reshape(target_shape=[-1, layer[2]])(x) print(x.get_shape().as_list()) x = CapsuleLayer(gene[index][7], gene[index][8], 3, name='digitcaps')(x) print(x.get_shape().as_list()) dim_capsule_out=gene[index][8] caps_index=caps_index+1 remaining=remaining-1 print(x.get_shape().as_list()) print("Added ClassCaps_layer\n") # DeepCaps Cells # d_caps = [2, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules] elif gene[index][0]==2 and remaining>1: if prevlayertype==0: x = ConvertToCaps()(x) print("ConvertToCaps"+str(prevlayertype)) print(x.get_shape().as_list()) x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(gene[index][5], gene[index][5]), r_num=1, b_alphas=[1, 1, 1])(x) deepcaps_index=deepcaps_index+1 if remaining==2: x_skip = ConvCapsuleLayer3D(kernel_size=3, num_capsule=gene[index][7], num_atoms=gene[index][8], strides=1, padding='same', routings=3)(x) deepcaps_index=deepcaps_index+1 else: x_skip = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x) deepcaps_index=deepcaps_index+1 x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x) deepcaps_index=deepcaps_index+1 x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x) x = layers.Add()([x, x_skip]) count +=1 #CapsCell count print("Added CapsCell_layer\n") if xtra_skip-1==count-1: x1 = x xtra_skip=-2 #flag==2 for inserted xtra skip elif remaining==1 and xtra_skip==-2: x2 = x elif remaining==2 and xtra_skip==-1: x1 = x x2 = x deepcaps_index=deepcaps_index+1 remaining=remaining-1 x2 = x prevlayertype=2 #FlattenCaps elif gene[index][0]==2 and remaining==1: print(x1.get_shape().as_list()) print(x2.get_shape().as_list()) flatCaps = FlattenCaps() xa = flatCaps(x2) flatCaps = FlattenCaps() xb = flatCaps(x1) x = layers.Concatenate(axis=-2)([xa, xb]) print(xa.get_shape().as_list()) x = DClassCaps(num_capsule=gene[index][7], dim_capsule=gene[index][8], routings=3, channels=0, name='digit_caps')(x) print("Added FlattenCaps_layer\n") print(x.get_shape().as_list()) dim_capsule_out=gene[index][8] else: break digitcaps = x # Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape. # If using tensorflow, this will not be necessary. :) out_caps = Length(name='capsnet')(digitcaps) # Decoder network. y = layers.Input(shape=(n_class,)) masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction # Shared Decoder model in training and prediction decoder = models.Sequential(name='decoder') # DeepCaps Decoder if input_shape == (32, 32, 3): # cifar10 resized decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16 decoder.add(layers.Reshape((8, 8, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) #decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(3, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(32, 32, 3), name='out_recon')) # 64, 64 in origial code elif input_shape == (64, 64, 3): # cifar 10 resized decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16 decoder.add(layers.Reshape((8, 8, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(3, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(64, 64, 3), name='out_recon')) elif input_shape == (28, 28, 1): # mnist decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16 decoder.add(layers.Reshape((7, 7, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(1, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(28, 28, 1), name='out_recon')) elif input_shape == (56, 56, 1): # mnist resized decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16 decoder.add(layers.Reshape((7, 7, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(4, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(56, 56, 1), name='out_recon')) elif input_shape == (56, 56, 3): # mnist resized decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16 decoder.add(layers.Reshape((7, 7, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(12, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.summary() decoder.add(layers.Reshape(target_shape=(56, 56, 3), name='out_recon')) elif input_shape == (64, 64, 1): # mnist resized for deepcaps (64x64 inputs) decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16 decoder.add(layers.Reshape((8, 8, 16))) decoder.add(layers.BatchNormalization(momentum=0.8)) decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same")) decoder.add(layers.Conv2DTranspose(1, (3, 3), strides=(1, 1), padding="same")) decoder.add(layers.Activation("relu")) decoder.add(layers.Reshape(target_shape=(64, 64, 1), name='out_recon')) else: raise NotImplementedError(f"Unknown decoder for shape {input_shape}") decoder.summary() # Models for training and evaluation (prediction) train_model = models.Model([inputs, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(inputs, [out_caps, decoder(masked)]) # manipulate model noise = layers.Input(shape=(n_class, dim_capsule_out)) noised_digitcaps = layers.Add()([digitcaps, noise]) masked_noised_y = Mask()([noised_digitcaps, y]) manipulate_model = models.Model([inputs, y, noise], decoder(masked_noised_y)) return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings): """ Defining the CapsNet :param input_shape: data shape, 3d, [width, height, channels] :param n_class: number of classes :param routings: number of routing iterations :return: Two Keras Models, the first one used for training, and the second one for evaluation. """ x = layers.Input(shape=input_shape) conv1 = layers.Conv2D(filters=64, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv1')(x) conv2 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv2')(conv1) conv3 = layers.Conv2D(filters=256, kernel_size=3, strides=2, padding='valid', activation='relu', name='conv3')(conv2) primarycaps = PrimaryCap(conv3, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid') digitcaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings, channels=32, name='digitcaps')(primarycaps) out_caps = Length(name='capsnet')(digitcaps) """ Decoder Network """ y = layers.Input(shape=(n_class, )) masked_by_y = Mask()([digitcaps, y]) masked = Mask()(digitcaps) decoder = models.Sequential(name='decoder') decoder.add( Dense(input_dim=16 * n_class, activation="relu", output_dim=7 * 7 * 32)) decoder.add(Reshape((7, 7, 32))) decoder.add(BatchNormalization(momentum=0.8)) decoder.add( layers.Deconvolution2D(32, 3, 3, subsample=(1, 1), border_mode='same', activation="relu")) decoder.add( layers.Deconvolution2D(16, 3, 3, subsample=(2, 2), border_mode='same', activation="relu")) decoder.add( layers.Deconvolution2D(8, 3, 3, subsample=(2, 2), border_mode='same', activation="relu")) decoder.add( layers.Deconvolution2D(4, 3, 3, subsample=(1, 1), border_mode='same', activation="relu")) decoder.add( layers.Deconvolution2D(1, 3, 3, subsample=(1, 1), border_mode='same', activation="sigmoid")) decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon')) """ Models for training and evaluation (prediction) """ train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)]) eval_model = models.Model(x, [out_caps, decoder(masked)]) return train_model, eval_model