def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
embed = layers.Embedding(output_dim=emb_output_dim,
input_dim=length,
trainable=emb_train)(x)
conv1 = layers.Conv1D(filters=conv1_filt,
kernel_size=conv1_kern,
strides=conv1_stride,
padding=conv1_pad,
activation=conv1_act,
name='conv1')(embed)
primarycaps = PrimaryCap(conv1,
dim_capsule=caps1_dim,
n_channels=caps1_n_channels,
kernel_size=caps1_kern,
strides=caps1_stride,
padding=caps1_pad)
digitcaps = CapsuleLayer(num_capsule=caps2_num,
dim_capsule=caps2_dim,
routings=routings,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
train_model = models.Model(x, out_caps)
return train_model
def build_discriminator():
img = Input(shape=img_shape)
# x = UpSampling2D()(x)
# x = Conv2D(128, kernel_size=3, strides=1, padding="same")(img)
# x = LeakyReLU(alpha=0.2)(x)
# x = BatchNormalization(momentum=0.8)(x)
x = Conv2D(256, kernel_size=9, strides=1, padding="valid")(img)
x = LeakyReLU(alpha=0.2)(x)
x = BatchNormalization(momentum=0.8)(x)
x = PrimaryCap(x,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
x = CapsuleLayer(num_capsule=10, dim_capsule=32, routings=routings)(x)
# x = Mask()(x)
# y = layers.Input(shape=(num_classes+1,))
# x2 = Mask()([x, y])
# # x2 = Length(name='capsnet')(x)
# x1 = Mask()(x)
x = Flatten()(x)
# Determine validity and label of the image
validity = Dense(1, activation="sigmoid", name='ds')(x)
label = Dense(num_classes, activation="softmax", name='label')(x)
return Model(img, [validity, label])
def CapsNet(input_shape, n_class, routings):
x = Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = Conv2D(filters=128,
kernel_size=9,
strides=2,
padding='valid',
activation='relu',
name='conv1')(x)
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=16,
kernel_size=9,
strides=4,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
out_caps = Flatten()(digitcaps)
# outputs = Dense(278)(out_caps)
# Models for training and evaluation (prediction)
train_model = models.Model(x, out_caps)
return train_model
def CapsNet(input_shape, n_class, num_routing, kernel_size=7):
"""
A Capsule Network on 2019ncon.
:param input_shape: data shape, 3d, [width, height, channels],for example:[21,28,1]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: A Keras Model with 2 inputs and 2 outputs
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=128, kernel_size=kernel_size, strides=1, padding='same', activation='relu', name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
primarycaps = PrimaryCap(conv1, dim_vector=16, n_channels=32, kernel_size=kernel_size, strides=2, padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=32, num_routing=num_routing, name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='out_caps')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer.
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(np.prod(input_shape), activation='sigmoid')(x_recon)
x_recon = layers.Reshape(target_shape=input_shape, name='out_recon')(x_recon)
# two-input-two-output keras Model
return models.Model([x, y], [out_caps, x_recon])
def CNN(seq_length, length, input_size, feature_maps, kernels, x):
concat_input = []
for feature_map, kernel in zip(feature_maps, kernels):
reduced_l = length - kernel + 1
#changes
# conv1 = Conv2D(feature_map, (1, kernel), activation='tanh', data_format="channels_last")(x)
conv1 = Conv2D(feature_map, (1, kernel),
strides=1,
activation='tanh',
data_format="channels_last")(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
primarycaps = PrimaryCap(conv1,
dim_vector=feature_map,
n_channels=32,
kernel_size=(1, kernel),
strides=2,
padding='valid',
filters=feature_map)
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=seq_length,
dim_vector=feature_map,
num_routing=3)(primarycaps)
# conv = Conv2D(feature_map, (1, kernel), activation='tanh', data_format="channels_last")(x)
# maxp = MaxPooling2D((1, reduced_l), data_format="channels_last")(conv)
concat_input.append(digitcaps)
x = Concatenate()(concat_input)
x = Reshape((seq_length, sum(feature_maps)))(x)
return x
def CapsNet(input_shape, n_class, num_routing):
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu')(x)
primarycaps = PrimaryCap(conv1,
dim_vector=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_vector=16,
num_routing=num_routing)(primarycaps)
out_caps = Length(name='out_caps')(digitcaps)
# Reconstruction network
y = layers.Input(shape=(n_class, ))
masked = Mask()([digitcaps, y])
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(784, activation='sigmoid')(x_recon)
x_recon = layers.Reshape(target_shape=[28, 28, 1],
name='out_recon')(x_recon)
return models.Model([x, y], [out_caps, x_recon])
def prepareModel():
"""
Capsule Network on biopsy patches.
"""
num_routing = 3
capsule_dimension = 8,
no_of_channels = 16
x = Input(shape=(224, 224, 3))
# Layer 1: Just a conventional Conv2D layer
conv1 = Conv2D(filters=32,
kernel_size=5,
strides=2,
padding='valid',
name='conv1')(x)
conv1 = Activation('relu', name='relu1')(conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=16,
kernel_size=5,
strides=2,
padding='valid')
primarycaps = Reshape((53 * 53, 16, 8))(primarycaps)
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = TimeDistributed(
CapsuleLayer(num_capsule=1,
dim_capsule=8,
num_routing=num_routing,
name='digitcaps'))(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
output = TrimmedAveragePool()(out_caps)
model = Model(inputs=x, outputs=output)
model.summary()
return model
def CapsNet(input_shape, n_class, num_routing):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 4d, [None, width, height, channels]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: A Keras Model with 2 inputs and 2 outputs
"""
x = layers.Input(shape=(maxlen,))
embed = layers.Embedding(max_features, embed_dim, input_length=maxlen)(x)
conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(
embed)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='out_caps')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer.
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon)
# two-input-two-output keras Model
return models.Model([x, y], [out_caps, x_recon])
def make_model():
x = Input(shape=input_shape)
conv1 = Conv3D(filters=conv_layer_filters, kernel_size=first_layer_kernel_size, strides=1,
padding='valid', activation='relu', name='conv1')(x)
primarycaps = PrimaryCap(conv1, dim_capsule=dim_primary_capsule, n_channels=n_channels,
kernel_size=primary_cap_kernel_size, strides=2, padding='valid',
name='primarycap_conv3d')
sub_caps = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_sub_capsule,
routings=3, name='sub_caps')(primarycaps)
out_caps = Length(name='capsnet')(sub_caps)
# Decoder network
y = Input(shape=(n_class,))
masked_by_y = Mask()([sub_caps, y])
masked = Mask()(sub_caps)
# shared decoder model in training and prediction
decoder = Sequential(name='decoder')
decoder.add(Dense(512, activation='relu',
input_dim=dim_sub_capsule*n_class))
decoder.add(Dense(1024, activation='relu'))
decoder.add(Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(Reshape(target_shape=input_shape, name='out_recon'))
### Models for training and evaluation (prediction and actually using)
train_model = Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = Model(x, [out_caps, decoder(masked)])
### manipulate model can be used to visualize activation maps for specific classes
noise = Input(shape=(n_class, dim_sub_capsule))
noised_sub_caps = Add()([sub_caps, noise])
masked_noised_y = Mask()([noised_sub_caps, y])
manipulate_model = Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings, model_version, lam_regularize):
"""
A Capsule Network .
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer, 对DEAP，kernel_size=9；对DREAMER，kernel_size=6
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1',
kernel_regularizer=regularizers.l2(lam_regularize))(
x) #kernel_size=9
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
# 对DEAP，kernel_size=9；对DREAMER，kernel_size=6
# 对CapsNet，strides=2，pading=‘valid’；对MLF-CapsNet，stides=1，padding='same'
if model_version == 'v0':
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid',
lam_regularize=lam_regularize,
model_version=model_version)
else:
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=1,
padding='same',
lam_regularize=lam_regularize,
model_version=model_version) #kernel_size=9
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps',
lam_regularize=lam_regularize)(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Models for training and evaluation (prediction)
y = layers.Input(shape=(n_class, ))
train_model = models.Model([x, y], out_caps)
eval_model = models.Model(x, out_caps)
return train_model, eval_model
def CapsNet(input_shape, n_class, routings, batch_size):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
out = models.Model(inputs=x, outputs=out_caps)
return out
def CapsNet(input_shape, n_class, num_routing):
"""
MNIST Veriseti için Kapsül Ağları.
:param input_shape: veri şekli, 3d, [genişlik, yükseklik, kanal]
:param n_class: sınıf sayısı
:param num_routing: dinamik yönlendirme (dynamic routing) iterasyon sayısı
:return: iki Keras model, birincisi eğitim için, ikincisi değerlendirme için (evalaution).
`eval_model` aynı zamanda eğitim için de kullanılabilir.
"""
x = layers.Input(shape=input_shape)
# Katman 1: Geleneksel Evrişimli Sinir Ağı Katmanı (Conv2D)
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Katman 2: Conv2D katmanı ile ezme (squash) aktivasyonu, [None, num_capsule, dim_capsule]’e yeniden şekil veriliyor.
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Katman 3: Kapsül Katmanı. Dinamik Yönlendirme algoritması burada çalışıyor.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Katman 4: Her kapsülün uzunluğunu yeniden düzenleyen yardımcı bir katmandır. Doğru etiketle eşleşmesi için bu işlem yapılır.
# Eğer Tensorflow kullanıyorsanız bu işleme gerek yoktur. :)
out_caps = Length(name='capsnet')(digitcaps)
# Kodçözücü Ağ.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # Doğru etiket, kapsül katmanın çıkışını maskelemek için kullanılır (Eğitim için).
masked = Mask()(
digitcaps
) # Filtre (maske), kapsülün maksimal uzunluğu ile kullanılır (Kestirim için).
# Eğitim ve Kestirimde Kodçözücü Modelin Paylaşımı
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Eğitim ve Değerlendirme (Kestirim) için Modeller
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu',
input_dim=16 * n_class)) # YES
decoder.add(layers.Dense(1024, activation='relu')) # YES
decoder.add(layers.Dense(4096, activation='relu')) # YES
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, num_routing):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
other_masks = []
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
# Layers 1: Convolution Layers to extract low level features
conv1 = layers.Conv2D(filters=64,
kernel_size=3,
strides=2,
activation='relu',
name='Conv1')(x)
conv2 = layers.Conv2D(filters=128,
kernel_size=3,
strides=2,
activation='relu',
name='Conv2')(conv1)
conv3 = layers.Conv2D(filters=256,
kernel_size=6,
strides=2,
activation='relu',
name='Conv3')(conv2)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv3,
dim_capsule=8,
n_channels=32,
kernel_size=6,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
classcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='classcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
out_caps = Length(name='capsnet')(classcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([classcaps, y])
masked = Mask()(classcaps)
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, routings):
"""
针对 MNIST 手写字符识别的胶囊网络
## input_shape: 输入数据的维度, 长度为3的列表, [width, height, channels]
## n_class: 类别的数量
## routings: routing算法迭代的次数
:return: 返回两个模型, 第一个用于训练, 第二个用于测试
"""
# Encoder network.输入为图片
x = layers.Input(shape=input_shape)
# Layer 1: 卷积层
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: PrimaryCap（自定义层）
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: CapsuleLayer（自定义胶囊层）
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: 模值计算层（自定义层）
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.输入为向量
y = layers.Input(shape=(n_class, ))
# 蒙版操作，对decoder输入的标准化
masked_by_y = Mask()([digitcaps, y]) # 用真实值取代胶囊层的输出值。此项用于训练
masked = Mask()(digitcaps) # 用最大长度值做胶囊的蒙版。此项用于评估
# 包含3层的全连接神经网络（序贯模型）
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# 最终将输出转为图片
# 训练模型和评估模型
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def build(self):
query = Input(name='query', shape=(self.config['text1_maxlen'],))
show_layer_info('Input', query)
doc = Input(name='doc', shape=(self.config['text2_maxlen'],))
show_layer_info('Input', doc)
dpool_index = Input(name='dpool_index', shape=[self.config['text1_maxlen'], self.config['text2_maxlen'], 3], dtype='int32')
show_layer_info('Input', dpool_index)
embedding = Embedding(self.config['vocab_size'], self.config['embed_size'], weights=[self.config['embed']], trainable = self.embed_trainable)
q_embed = embedding(query)
show_layer_info('Embedding', q_embed)
d_embed = embedding(doc)
show_layer_info('Embedding', d_embed)
cross = Dot(axes=[2, 2], normalize=False)([q_embed, d_embed])
show_layer_info('Dot', cross)
cross_reshape = Reshape((self.config['text1_maxlen'], self.config['text2_maxlen'], 1))(cross)
show_layer_info('Reshape', cross_reshape)
conv2d = Conv2D(self.config['kernel_count'], self.config['kernel_size'], padding='same', activation='relu')
dpool = DynamicMaxPooling(self.config['dpool_size'][0], self.config['dpool_size'][1])
conv1 = conv2d(cross_reshape)
show_layer_info('Conv2D', conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=2, dim_capsule=16, routings=3,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
pool1 = dpool([conv1, dpool_index])
show_layer_info('DynamicMaxPooling', pool1)
pool1_flat = Flatten()(pool1)
show_layer_info('Flatten', pool1_flat)
pool1_flat_drop = Dropout(rate=self.config['dropout_rate'])(pool1_flat)
show_layer_info('Dropout', pool1_flat_drop)
if self.config['target_mode'] == 'classification':
out_ = Dense(2, activation='softmax')(pool1_flat_drop)
elif self.config['target_mode'] in ['regression', 'ranking']:
out_ = Dense(1)(pool1_flat_drop)
show_layer_info('Dense', out_)
model = Model(inputs=[query, doc, dpool_index], outputs=out_)
model.summary()
return model
def CapsNet_NoDecoder(input_shape, n_class, kernel, primary_channel,
primary_veclen, digit_veclen, dropout, routings,
model_size_info):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
model_size_info: kernel, conv_channel, primary_channel, input_veclen, output_veclen
"""
kernel, conv_channel, primary_channel, primary_veclen, digit_veclen = model_size_info
if kernel > 19:
conv_padding = 'same'
print('kernel size big, padding to same')
else:
conv_padding = 'valid'
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=kernel,
strides=1,
padding=conv_padding,
activation='relu',
name='conv1')(x)
if dropout != 0:
conv1 = layers.Dropout(dropout)(conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=primary_veclen,
n_channels=primary_channel,
kernel_size=kernel,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=digit_veclen,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Models for training and evaluation (prediction)
train_model = models.Model(x, out_caps)
return train_model
def build_model(input_shape):
# encoder
x = Input(shape=input_shape)
conv1 = Conv2D(filters=64,
kernel_size=3,
padding='same',
trainable=True,
name='conv1')(x)
conv1 = BatchNormalization()(conv1)
conv1 = Activation('relu')(conv1)
conv1 = Conv2D(filters=64,
kernel_size=3,
padding='same',
trainable=True,
name='conv2')(conv1)
conv1 = BatchNormalization()(conv1)
conv1 = Activation('relu')(conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=N_CLASS,
dim_capsule=16,
routings=ROUTINGS,
name='digitcaps')(primarycaps)
digitcaps = Flatten()(digitcaps)
decoder = Dense(512, activation='relu', input_dim=16 * N_CLASS)(digitcaps)
decoder = Dense(1024, activation='relu')(decoder)
decoder = Dense(np.prod(input_shape), activation='sigmoid')(decoder)
decoder = Reshape(input_shape)(decoder)
model = Model(x, decoder)
# transfer weights from first 2 layers of VGG-19
weights = pre_trained_model.layers[1].get_weights()[0][:, :, :, :]
#weights = np.reshape(weights, (3,3,1,64))
bias = pre_trained_model.layers[1].get_weights()[1]
model.layers[1].set_weights([weights, bias])
weights = pre_trained_model.layers[2].get_weights()[0]
bias = pre_trained_model.layers[2].get_weights()[1]
model.layers[4].set_weights([weights, bias])
return model
def CapsNet(input_shape, n_class, routings):
left_input = layers.Input(shape=input_shape)
right_input = layers.Input(shape=input_shape)
input = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(input)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
tunnel_l = models.Model(input, out_caps)
tunnel_r = models.Model(input, out_caps)
encoded_l = tunnel_l(left_input)
encoded_r = tunnel_r(right_input)
L1_layer = layers.Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
L1_distance = L1_layer([encoded_l, encoded_r])
prediction = layers.Dense(1, activation='sigmoid')(L1_distance)
train_model = models.Model(inputs=[left_input, right_input],
outputs=prediction)
return train_model
def CapsNet(input_shape, n_class, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(
filters=256,
kernel_size=9,
strides=4, # TODO: use 5
padding='valid',
activation='relu',
name='conv1')(
x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(
conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(
num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(
primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Models for training and evaluation (prediction)
model = models.Model(x, out_caps)
return model
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=256,
kernel_size=(9),
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y])
masked = Mask()(digitcaps)
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings, batch_size):
# input layer
x = tf.keras.layers.Input(input_shape, batch_size = batch_size)
# layer 1 : regular Convolutionnal layer
conv1 = tf.keras.layers.Conv2D(filters = 256, kernel_size = (9,9), activation = 'relu', name = 'conv1')(x)
# layer 2 : PrimaryCaps, which is a convolution layer with Squash activation
# dim_capsule : corresponds to the dimension of the capsule output vector
# n_channels : number of capsule types
primarycaps = PrimaryCap(conv1, dim_capsule = 8, n_channels = 32, kernel_size = 9, strides = 2, padding = 'valid')
# layer 3 : CapsuleLayer (involves routing by agreement)
# each capsule in this layer represents one of the Kuzushiji symbol
kcaps = CapsuleLayer(num_capsule = n_class, dim_capsule = 16, routings = routings, name = 'kcaps')(primarycaps)
# layer 4 : layer that takes the length of each capsule
out_caps = Length(name='capsnet')(kcaps)
# Let's build the decoder network
# 2 reconstructions are performed :
# - first one is to reconstruct image according to the true label
# - second one is to reconstruct image according to the vector with maximal length (prediction)
y = tf.keras.layers.Input((n_class,))
masked_by_y = Mask()([kcaps, y])
masked = Mask()(kcaps)
# Dense layers of the decoder architecture as described in the paper
decoder = tf.keras.models.Sequential(name = 'decoder')
decoder.add(tf.keras.layers.Dense(512, activation = 'relu', input_dim = 16 * n_class))
decoder.add(tf.keras.layers.Dense(1024, activation = 'relu'))
decoder.add(tf.keras.layers.Dense(input_shape[0]*input_shape[1], activation = 'sigmoid'))
decoder.add(tf.keras.layers.Reshape(input_shape, name = 'out_recon'))
# Models used for training and evaluation
# train_model involves training of the decoder
# while evaluation model, given an input x, outputs his prediction and his reconstruction using the trained decoder
train_model = tf.keras.models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = tf.keras.models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def create_layers(input_img, n_class, routings):
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=128,
kernel_size=5,
strides=1,
padding='valid',
activation='relu')(input_img)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=12,
kernel_size=5,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=4,
routings=routings)(primarycaps)
return digitcaps
def create_layers(input_img, n_class, routings):
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=5,
strides=1,
padding='valid',
activation='relu')(input_img)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=5,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings)(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
return digitcaps
def CapsNet(input_shape, n_class, routings):
"""
MNISTに関するカプセルネットワーク
:param input_shape: 入力データのshape(3次元で[width, height, channels]という形)
:param n_class: クラスの数
:param routings: routingを行う回数
:return 2つのmodel (1つ目:学習用モデル, 2つ目:評価用モデル)
`eval_model`というモデルも学習用としてしようすることもできる.
"""
x = layers.Input(shape=input_shape)
# 1層目: ただの2次元畳み込み層
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# 2層目: 活性化関数にsquash関数を用いた2次元畳み込み層で,[None ,num_capsule, dim_capsule]という形に変換する
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# 3層目: カプセル層 (routingアルゴリズムはここで行っている)
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# 4層目: ここはカプセルを"長さ"に変形するための補助レイヤーで, 教師データの形に合わせている.
# tensorflowを使用している場合, ここは必要ありません.
out_caps = Length(name='capsnet')(digitcaps)
# Decoderネットワーク
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y]) # 正解のラベルはよく学習のためにカプセル層の出力を隠す
masked = Mask()(digitcaps) # マスクは予測のために長さが最大値のカプセルを使用する
# このDecoderモデルは学習時にも予測時にも使われる.
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# 学習モデルと評価モデル
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# モデルにノイズを加える
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def MultiLevelDCNet(input_shape, n_class, routings):
"""
A Multi-level DCNet on CIFAR-10.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
"""
x = layers.Input(shape=input_shape)
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
########################### Level 1 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(x, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 1)
# Here PrimaryCapsConv2D is the Conv2D output which is used as the primary capsules by reshaping and squashing (squash activation).
# primarycaps_1 (size: [None, num_capsule, dim_capsule]) is the "reshaped and sqashed output" which will be further passed to the dynamic routing protocol.
primarycaps_1, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid')
# Applying ReLU Activation to primary capsules
conv = layers.Activation('relu')(PrimaryCapsConv2D)
########################### Level 2 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 2)
primarycaps_2, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid')
# Applying ReLU Activation to primary capsules
conv = layers.Activation('relu')(PrimaryCapsConv2D)
########################### Level 3 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 3)
primarycaps_3, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=3, strides=2, padding='valid')
# Merging Primary Capsules for the Merged DigitCaps (CapsuleLayer formed by combining all levels of primary capsules)
mergedLayer = layers.merge([primarycaps_1,primarycaps_2,primarycaps_3], mode='concat', concat_axis=1)
########################### Separate DigitCaps Outputs (used for training) ###########################
# Merged DigitCaps
digitcaps_0 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
name='digitcaps0')(mergedLayer)
out_caps_0 = Length(name='capsnet_0')(digitcaps_0)
# First Level DigitCaps
digitcaps_1 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
name='digitcaps1')(primarycaps_1)
out_caps_1 = Length(name='capsnet_1')(digitcaps_1)
# Second Level DigitCaps
digitcaps_2 = CapsuleLayer(num_capsule=n_class, dim_capsule=12, routings=routings,
name='digitcaps2')(primarycaps_2)
out_caps_2 = Length(name='capsnet_2')(digitcaps_2)
# Third Level DigitCaps
digitcaps_3 = CapsuleLayer(num_capsule=n_class, dim_capsule=10, routings=routings,
name='digitcaps3')(primarycaps_3)
out_caps_3 = Length(name='capsnet_3')(digitcaps_3)
########################### Combined DigitCaps Output (used for evaluation) ###########################
digitcaps = layers.merge([digitcaps_1,digitcaps_2,digitcaps_3, digitcaps_0], mode='concat', concat_axis=2,
name='digitcaps')
out_caps = Length(name='capsnet')(digitcaps)
# Reconstruction (decoder) network
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(600, activation='relu', input_dim=int(digitcaps.shape[2]*n_class), name='zero_layer'))
decoder.add(layers.Dense(600, activation='relu', name='one_layer'))
decoderFinal = models.Sequential(name='decoderFinal')
# Concatenating two layers
decoderFinal.add(layers.Merge([decoder.get_layer('zero_layer'), decoder.get_layer('one_layer')], mode='concat'))
decoderFinal.add(layers.Dense(1200, activation='relu'))
decoderFinal.add(layers.Dense(np.prod([32,32,1]), activation='sigmoid'))
decoderFinal.add(layers.Reshape(target_shape=[32,32,1], name='out_recon'))
# Model for training
train_model = models.Model([x, y], [out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked_by_y)])
# Model for evaluation (prediction)
# Note that out_caps is the final prediction. Other predictions could be used for analysing separate-level predictions.
eval_model = models.Model(x, [out_caps, out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked)])
return train_model, eval_model
def capsModel(input_shape, n_class, num_routing):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=512,
kernel_size=7,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
BN1 = layers.BatchNormalization()(conv1)
conv2 = layers.Conv2D(filters=256,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv2')(BN1)
BN2 = layers.BatchNormalization()(conv2)
conv3 = layers.Conv2D(filters=256,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv3')(BN2)
BN3 = layers.BatchNormalization()(conv3)
conv4 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv4')(BN3)
BN4 = layers.BatchNormalization()(conv4)
conv5 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv5')(BN4)
BN5 = layers.BatchNormalization()(conv5)
conv6 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv6')(BN5)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv6,
dim_capsule=24,
n_channels=32,
kernel_size=12,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(gene, input_shape, n_class, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
inputs = layers.Input(shape=input_shape)
x = inputs
print("### Input shape", )
# if input_shape==(28, 28, 1):
# dataset="mnist"
# elif input_shape==(32, 32, 3):
# dataset="cifar"
# elif input_shape==(32, 32, 3) and gene[len(gene)-1][0]==2:
# dataset="cifar_resized"
remaining=len(gene)-2
conv_index=1
caps_index=1
deepcaps_index=1
count=0
xtra_skip=gene[len(gene)-2][0]
print("xtra_skip is " +str(gene[len(gene)-2][0])+"\n")
# Scan for type of layers
#N_layers = int((len(gene)-2)/9) # -1 for xtraskip term
conv=[]
caps=[]
dcaps=[]
N_conv=0
N_caps=0
N_dcaps=0
i=0
prevlayertype=0
for index in range(0,len(gene)):
if gene[index][0]==0:
N_conv = N_conv+1
conv.append(i)
i+=1
elif gene[index][0]==1:
type=1
N_caps = N_caps+1
caps.append(i)
i+=1
elif gene[index][0]==2:
convert=1
type=2
N_dcaps = N_dcaps+1
dcaps.append(i)
i+=1
for index in range(0,len(gene)):
#for layer in gene:
if len(gene[index])>1:
# Convolutional layers
# conv = [0, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules]
if gene[index][0]==0:
if prevlayertype==2:
x=layers.Reshape((gene[index][1], gene[index][1], -1))(x)
print("ConvertToConv"+str(prevlayertype))
x = layers.Conv2D(filters=gene[index][7]*gene[index][8], kernel_size=gene[index][4], strides=gene[index][5], padding='same', data_format="channels_last")(x)
x = BatchNormalization()(x)
print(x.get_shape().as_list())
conv_index=conv_index+1
remaining=remaining-1
print("Added Conv%s_layer" % str(conv_index-1))
count +=1
if xtra_skip==count-1 or (xtra_skip==0 and count==1):
x1 = x
xtra_skip=-2 #flag==2 for inserted xtra skip
prevlayertype=0
# Primary Capsules layers
# caps = [1, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules]
elif gene[index][0]==1 and remaining>1:
x = PrimaryCap(x, dim_capsule=gene[index][8], n_channels=gene[index][7], kernel_size=gene[index][4], strides=gene[index][5], padding='same') #CapsuleLayer(layer[7], layer[8])(x)
print(x.get_shape().as_list())
caps_index=caps_index+1
remaining=remaining-1
print("rem "+str(remaining))
print("Added Caps%s_layer" % str(caps_index-1))
veclen=gene[index][7]
numout=gene[index][8]
prevlayertype=1
# Class Capsules layer
# caps = [1, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules]
elif gene[index][0]==1 and remaining==1:
print(x.get_shape().as_list())
x = layers.Reshape(target_shape=[-1, gene[index][2]])(x) #[veclen*layer[1]*layer[1], numout])(x) # added this to solve error instead of Reshape(target_shape=[-1, layer[2]])(x)
print(x.get_shape().as_list())
x = CapsuleLayer(gene[index][7], gene[index][8], 3, name='digitcaps')(x)
print(x.get_shape().as_list())
dim_capsule_out=gene[index][8]
caps_index=caps_index+1
remaining=remaining-1
print(x.get_shape().as_list())
print("Added ClassCaps_layer\n")
# DeepCaps Cells
# d_caps = [2, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules]
elif gene[index][0]==2 and remaining>1:
if prevlayertype==0:
x = ConvertToCaps()(x)
print("ConvertToCaps"+str(prevlayertype))
print(x.get_shape().as_list())
x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(gene[index][5], gene[index][5]), r_num=1, b_alphas=[1, 1, 1])(x)
deepcaps_index=deepcaps_index+1
if remaining==2:
x_skip = ConvCapsuleLayer3D(kernel_size=3, num_capsule=gene[index][7], num_atoms=gene[index][8], strides=1, padding='same', routings=3)(x)
deepcaps_index=deepcaps_index+1
else:
x_skip = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x)
deepcaps_index=deepcaps_index+1
x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x)
deepcaps_index=deepcaps_index+1
x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x)
x = layers.Add()([x, x_skip])
count +=1 #CapsCell count
print("Added CapsCell_layer\n")
if xtra_skip-1==count-1:
x1 = x
xtra_skip=-2 #flag==2 for inserted xtra skip
elif remaining==1 and xtra_skip==-2:
x2 = x
elif remaining==2 and xtra_skip==-1:
x1 = x
x2 = x
deepcaps_index=deepcaps_index+1
remaining=remaining-1
x2 = x
prevlayertype=2
#FlattenCaps
elif gene[index][0]==2 and remaining==1:
print(x1.get_shape().as_list())
print(x2.get_shape().as_list())
flatCaps = FlattenCaps()
xa = flatCaps(x2)
flatCaps = FlattenCaps()
xb = flatCaps(x1)
x = layers.Concatenate(axis=-2)([xa, xb])
print(xa.get_shape().as_list())
x = DClassCaps(num_capsule=gene[index][7], dim_capsule=gene[index][8], routings=3, channels=0, name='digit_caps')(x)
print("Added FlattenCaps_layer\n")
print(x.get_shape().as_list())
dim_capsule_out=gene[index][8]
else:
break
digitcaps = x
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
# DeepCaps Decoder
if input_shape == (32, 32, 3): # cifar10 resized
decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16
decoder.add(layers.Reshape((8, 8, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
#decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(3, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(32, 32, 3), name='out_recon')) # 64, 64 in origial code
elif input_shape == (64, 64, 3): # cifar 10 resized
decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16
decoder.add(layers.Reshape((8, 8, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(3, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(64, 64, 3), name='out_recon'))
elif input_shape == (28, 28, 1): # mnist
decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16
decoder.add(layers.Reshape((7, 7, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(1, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(28, 28, 1), name='out_recon'))
elif input_shape == (56, 56, 1): # mnist resized
decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16
decoder.add(layers.Reshape((7, 7, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(4, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(56, 56, 1), name='out_recon'))
elif input_shape == (56, 56, 3): # mnist resized
decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16
decoder.add(layers.Reshape((7, 7, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(12, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.summary()
decoder.add(layers.Reshape(target_shape=(56, 56, 3), name='out_recon'))
elif input_shape == (64, 64, 1): # mnist resized for deepcaps (64x64 inputs)
decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16
decoder.add(layers.Reshape((8, 8, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(1, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(64, 64, 1), name='out_recon'))
else:
raise NotImplementedError(f"Unknown decoder for shape {input_shape}")
decoder.summary()
# Models for training and evaluation (prediction)
train_model = models.Model([inputs, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(inputs, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, dim_capsule_out))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([inputs, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings):
"""
Defining the CapsNet
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
"""
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=64,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
conv2 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv2')(conv1)
conv3 = layers.Conv2D(filters=256,
kernel_size=3,
strides=2,
padding='valid',
activation='relu',
name='conv3')(conv2)
primarycaps = PrimaryCap(conv3,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
channels=32,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
"""
Decoder Network
"""
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y])
masked = Mask()(digitcaps)
decoder = models.Sequential(name='decoder')
decoder.add(
Dense(input_dim=16 * n_class, activation="relu",
output_dim=7 * 7 * 32))
decoder.add(Reshape((7, 7, 32)))
decoder.add(BatchNormalization(momentum=0.8))
decoder.add(
layers.Deconvolution2D(32,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(16,
3,
3,
subsample=(2, 2),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(8,
3,
3,
subsample=(2, 2),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(4,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(1,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="sigmoid"))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
"""
Models for training and evaluation (prediction)
"""
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
