def make_model():
x = Input(shape=input_shape)
conv1 = Conv3D(filters=conv_layer_filters, kernel_size=first_layer_kernel_size, strides=1,
padding='valid', activation='relu', name='conv1')(x)
primarycaps = PrimaryCap(conv1, dim_capsule=dim_primary_capsule, n_channels=n_channels,
kernel_size=primary_cap_kernel_size, strides=2, padding='valid',
name='primarycap_conv3d')
sub_caps = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_sub_capsule,
routings=3, name='sub_caps')(primarycaps)
out_caps = Length(name='capsnet')(sub_caps)
# Decoder network
y = Input(shape=(n_class,))
masked_by_y = Mask()([sub_caps, y])
masked = Mask()(sub_caps)
# shared decoder model in training and prediction
decoder = Sequential(name='decoder')
decoder.add(Dense(512, activation='relu',
input_dim=dim_sub_capsule*n_class))
decoder.add(Dense(1024, activation='relu'))
decoder.add(Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(Reshape(target_shape=input_shape, name='out_recon'))
### Models for training and evaluation (prediction and actually using)
train_model = Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = Model(x, [out_caps, decoder(masked)])
### manipulate model can be used to visualize activation maps for specific classes
noise = Input(shape=(n_class, dim_sub_capsule))
noised_sub_caps = Add()([sub_caps, noise])
masked_noised_y = Mask()([noised_sub_caps, y])
manipulate_model = Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, num_routing):
"""
MNIST Veriseti için Kapsül Ağları.
:param input_shape: veri şekli, 3d, [genişlik, yükseklik, kanal]
:param n_class: sınıf sayısı
:param num_routing: dinamik yönlendirme (dynamic routing) iterasyon sayısı
:return: iki Keras model, birincisi eğitim için, ikincisi değerlendirme için (evalaution).
`eval_model` aynı zamanda eğitim için de kullanılabilir.
"""
x = layers.Input(shape=input_shape)
# Katman 1: Geleneksel Evrişimli Sinir Ağı Katmanı (Conv2D)
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Katman 2: Conv2D katmanı ile ezme (squash) aktivasyonu, [None, num_capsule, dim_capsule]’e yeniden şekil veriliyor.
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Katman 3: Kapsül Katmanı. Dinamik Yönlendirme algoritması burada çalışıyor.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Katman 4: Her kapsülün uzunluğunu yeniden düzenleyen yardımcı bir katmandır. Doğru etiketle eşleşmesi için bu işlem yapılır.
# Eğer Tensorflow kullanıyorsanız bu işleme gerek yoktur. :)
out_caps = Length(name='capsnet')(digitcaps)
# Kodçözücü Ağ.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # Doğru etiket, kapsül katmanın çıkışını maskelemek için kullanılır (Eğitim için).
masked = Mask()(
digitcaps
) # Filtre (maske), kapsülün maksimal uzunluğu ile kullanılır (Kestirim için).
# Eğitim ve Kestirimde Kodçözücü Modelin Paylaşımı
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Eğitim ve Değerlendirme (Kestirim) için Modeller
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, num_routing):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
other_masks = []
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu',
input_dim=16 * n_class)) # YES
decoder.add(layers.Dense(1024, activation='relu')) # YES
decoder.add(layers.Dense(4096, activation='relu')) # YES
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
# Layers 1: Convolution Layers to extract low level features
conv1 = layers.Conv2D(filters=64,
kernel_size=3,
strides=2,
activation='relu',
name='Conv1')(x)
conv2 = layers.Conv2D(filters=128,
kernel_size=3,
strides=2,
activation='relu',
name='Conv2')(conv1)
conv3 = layers.Conv2D(filters=256,
kernel_size=6,
strides=2,
activation='relu',
name='Conv3')(conv2)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv3,
dim_capsule=8,
n_channels=32,
kernel_size=6,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
classcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='classcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
out_caps = Length(name='capsnet')(classcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([classcaps, y])
masked = Mask()(classcaps)
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, routings):
"""
针对 MNIST 手写字符识别的胶囊网络
## input_shape: 输入数据的维度, 长度为3的列表, [width, height, channels]
## n_class: 类别的数量
## routings: routing算法迭代的次数
:return: 返回两个模型, 第一个用于训练, 第二个用于测试
"""
# Encoder network.输入为图片
x = layers.Input(shape=input_shape)
# Layer 1: 卷积层
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: PrimaryCap（自定义层）
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: CapsuleLayer（自定义胶囊层）
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: 模值计算层（自定义层）
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.输入为向量
y = layers.Input(shape=(n_class, ))
# 蒙版操作，对decoder输入的标准化
masked_by_y = Mask()([digitcaps, y]) # 用真实值取代胶囊层的输出值。此项用于训练
masked = Mask()(digitcaps) # 用最大长度值做胶囊的蒙版。此项用于评估
# 包含3层的全连接神经网络（序贯模型）
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# 最终将输出转为图片
# 训练模型和评估模型
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, num_routing):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 4d, [None, width, height, channels]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: A Keras Model with 2 inputs and 2 outputs
"""
x = layers.Input(shape=(maxlen,))
embed = layers.Embedding(max_features, embed_dim, input_length=maxlen)(x)
conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(
embed)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='out_caps')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer.
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon)
# two-input-two-output keras Model
return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, num_routing):
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu')(x)
primarycaps = PrimaryCap(conv1,
dim_vector=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_vector=16,
num_routing=num_routing)(primarycaps)
out_caps = Length(name='out_caps')(digitcaps)
# Reconstruction network
y = layers.Input(shape=(n_class, ))
masked = Mask()([digitcaps, y])
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(784, activation='sigmoid')(x_recon)
x_recon = layers.Reshape(target_shape=[28, 28, 1],
name='out_recon')(x_recon)
return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, num_routing, kernel_size=7):
"""
A Capsule Network on 2019ncon.
:param input_shape: data shape, 3d, [width, height, channels],for example:[21,28,1]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: A Keras Model with 2 inputs and 2 outputs
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=128, kernel_size=kernel_size, strides=1, padding='same', activation='relu', name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
primarycaps = PrimaryCap(conv1, dim_vector=16, n_channels=32, kernel_size=kernel_size, strides=2, padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=32, num_routing=num_routing, name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='out_caps')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer.
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(np.prod(input_shape), activation='sigmoid')(x_recon)
x_recon = layers.Reshape(target_shape=input_shape, name='out_recon')(x_recon)
# two-input-two-output keras Model
return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=256,
kernel_size=(9),
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y])
masked = Mask()(digitcaps)
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings, batch_size):
# input layer
x = tf.keras.layers.Input(input_shape, batch_size = batch_size)
# layer 1 : regular Convolutionnal layer
conv1 = tf.keras.layers.Conv2D(filters = 256, kernel_size = (9,9), activation = 'relu', name = 'conv1')(x)
# layer 2 : PrimaryCaps, which is a convolution layer with Squash activation
# dim_capsule : corresponds to the dimension of the capsule output vector
# n_channels : number of capsule types
primarycaps = PrimaryCap(conv1, dim_capsule = 8, n_channels = 32, kernel_size = 9, strides = 2, padding = 'valid')
# layer 3 : CapsuleLayer (involves routing by agreement)
# each capsule in this layer represents one of the Kuzushiji symbol
kcaps = CapsuleLayer(num_capsule = n_class, dim_capsule = 16, routings = routings, name = 'kcaps')(primarycaps)
# layer 4 : layer that takes the length of each capsule
out_caps = Length(name='capsnet')(kcaps)
# Let's build the decoder network
# 2 reconstructions are performed :
# - first one is to reconstruct image according to the true label
# - second one is to reconstruct image according to the vector with maximal length (prediction)
y = tf.keras.layers.Input((n_class,))
masked_by_y = Mask()([kcaps, y])
masked = Mask()(kcaps)
# Dense layers of the decoder architecture as described in the paper
decoder = tf.keras.models.Sequential(name = 'decoder')
decoder.add(tf.keras.layers.Dense(512, activation = 'relu', input_dim = 16 * n_class))
decoder.add(tf.keras.layers.Dense(1024, activation = 'relu'))
decoder.add(tf.keras.layers.Dense(input_shape[0]*input_shape[1], activation = 'sigmoid'))
decoder.add(tf.keras.layers.Reshape(input_shape, name = 'out_recon'))
# Models used for training and evaluation
# train_model involves training of the decoder
# while evaluation model, given an input x, outputs his prediction and his reconstruction using the trained decoder
train_model = tf.keras.models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = tf.keras.models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet_nogradientstop_crossentropy(input_shape, n_class, routings): # best testing results! val 0.13xx testX cnn1 200 1 cnn2 150 9 drop1 0.68 drop20.68 n_channels 50 kernel_size 20,dropout1
x = layers.Input(shape=input_shape)
conv1 = layers.Conv1D(filters=200, kernel_size=1, strides=1, padding='valid', kernel_initializer='he_normal',activation='relu', name='conv1')(x)
#conv1=BatchNormalization()(conv1)
conv1 = Dropout(0.7)(conv1)
conv2 = layers.Conv1D(filters=200, kernel_size=9, strides=1, padding='valid', kernel_initializer='he_normal',activation='relu', name='conv2')(conv1)
#conv1=BatchNormalization()(conv1)
conv2 = Dropout(0.75)(conv2) #0.75 valx loss has 0.1278!
primarycaps = PrimaryCap(conv2, dim_capsule=8, n_channels=60, kernel_size=20, kernel_initializer='he_normal',strides=1, padding='valid',dropout=0.2)
dim_capsule_dim2=10
# Capsule layer. Routing algorithm works here.
digitcaps_c = CapsuleLayer_nogradient_stop(num_capsule=n_class, dim_capsule=dim_capsule_dim2, num_routing=routings,name='digitcaps',kernel_initializer='he_normal',dropout=0.1)(primarycaps)
#digitcaps_c = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_capsule_dim2, num_routing=routings,name='digitcaps',kernel_initializer='he_normal')(primarycaps)
digitcaps = Extract_outputs(dim_capsule_dim2)(digitcaps_c)
weight_c = Extract_weight_c(dim_capsule_dim2)(digitcaps_c)
out_caps = Length()(digitcaps)
out_caps = Activation('softmax',name='capsnet')(out_caps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=dim_capsule_dim2*n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = Model(x, [out_caps, decoder(masked)])
weight_c_model = Model(x,weight_c)
# manipulate model
noise = layers.Input(shape=(n_class, dim_capsule_dim2))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model,weight_c_model
def CapsNet(input_shape, n_class, routings):
"""
Defining the CapsNet
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
"""
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=64,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
conv2 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv2')(conv1)
conv3 = layers.Conv2D(filters=256,
kernel_size=3,
strides=2,
padding='valid',
activation='relu',
name='conv3')(conv2)
primarycaps = PrimaryCap(conv3,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
channels=32,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
"""
Decoder Network
"""
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y])
masked = Mask()(digitcaps)
decoder = models.Sequential(name='decoder')
decoder.add(
Dense(input_dim=16 * n_class, activation="relu",
output_dim=7 * 7 * 32))
decoder.add(Reshape((7, 7, 32)))
decoder.add(BatchNormalization(momentum=0.8))
decoder.add(
layers.Deconvolution2D(32,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(16,
3,
3,
subsample=(2, 2),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(8,
3,
3,
subsample=(2, 2),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(4,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="relu"))
decoder.add(
layers.Deconvolution2D(1,
3,
3,
subsample=(1, 1),
border_mode='same',
activation="sigmoid"))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
"""
Models for training and evaluation (prediction)
"""
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNetBasic(input_shape, n_class=2):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=5,
strides=1,
padding='same',
activation='relu',
name='conv1')(x)
# Reshape layer to be 1 capsule x [filters] atoms
_, H, W, C = conv1.get_shape()
conv1_reshaped = layers.Reshape((H.value, W.value, 1, C.value))(conv1)
# Layer 1: Primary Capsule: Conv cap with routing 1
primary_caps = ConvCapsuleLayer(kernel_size=5,
num_capsule=8,
num_atoms=32,
strides=1,
padding='same',
routings=1,
name='primarycaps')(conv1_reshaped)
# Layer 4: Convolutional Capsule: 1x1
seg_caps = ConvCapsuleLayer(kernel_size=1,
num_capsule=1,
num_atoms=16,
strides=1,
padding='same',
routings=3,
name='seg_caps')(primary_caps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
out_seg = Length(num_classes=n_class, seg=True, name='out_seg')(seg_caps)
# Decoder network.
_, H, W, C, A = seg_caps.get_shape()
y = layers.Input(shape=input_shape[:-1] + (1, ))
masked_by_y = Mask()(
[seg_caps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(
seg_caps) # Mask using the capsule with maximal length. For prediction
def shared_decoder(mask_layer):
recon_remove_dim = layers.Reshape(
(H.value, W.value, A.value))(mask_layer)
recon_1 = layers.Conv2D(filters=64,
kernel_size=1,
padding='same',
kernel_initializer='he_normal',
activation='relu',
name='recon_1')(recon_remove_dim)
recon_2 = layers.Conv2D(filters=128,
kernel_size=1,
padding='same',
kernel_initializer='he_normal',
activation='relu',
name='recon_2')(recon_1)
out_recon = layers.Conv2D(filters=1,
kernel_size=1,
padding='same',
kernel_initializer='he_normal',
activation='sigmoid',
name='out_recon')(recon_2)
return out_recon
# Models for training and evaluation (prediction)
train_model = models.Model(inputs=[x, y],
outputs=[out_seg,
shared_decoder(masked_by_y)])
eval_model = models.Model(inputs=x,
outputs=[out_seg,
shared_decoder(masked)])
# manipulate model
noise = layers.Input(shape=((H.value, W.value, C.value, A.value)))
noised_seg_caps = layers.Add()([seg_caps, noise])
masked_noised_y = Mask()([noised_seg_caps, y])
manipulate_model = models.Model(inputs=[x, y, noise],
outputs=shared_decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
kernel_size=9,
strides=2,
padding="valid")
# Capsule Layer with routing algorithm
digitcaps = CapsuleLayer(num_capsule=num_classes,
dim_capsule=16,
routings=num_routing,
name="digitcaps")(primarycaps)
# Auxilliary layer to replace each capsule with its length
out_caps = Length(name="capsnet")(digitcaps)
# Building the Decoder Network
y = Input(shape=(num_classes, ))
masked_by_y = Mask()([digitcaps, y]) # True label used to mask the output
masked = Mask()(digitcaps) # Mask using the capsule with maximum length
# Shared Decoder model in training and prediction
decoder = Sequential(name='decoder')
decoder.add(Dense(512, activation="relu", input_dim=16 * num_classes))
decoder.add(Dense(1024, activation="relu"))
decoder.add(Dense(np.prod(x_train.shape[1:]), activation="sigmoid"))
decoder.add(Reshape(target_shape=input_shape, name="out_recon"))
# Models for training
model = Model([x, y], [out_caps, decoder(masked_by_y)])
def margin_loss(y_true, y_pred):
"""
def CapsNet(input_shape, n_class, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
poses_list = []
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256, kernel_size=3, strides=1, padding='same', activation='relu', name='conv1_1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps1 = PrimaryCap1(conv1, dim_capsule=8, n_channels=32, kernel_size=(5, 5), strides=2, padding='same')
#y Layer 3: Capsule layer. Routing algorithm works here.
digitcaps1 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
name='digitcaps1_1')(primarycaps1)
poses_list.append(digitcaps1)
# Layer 1: Just a conventional Conv2D layer
conv2 = layers.Conv2D(filters=256, kernel_size=5, strides=1, padding='same', activation='relu', name='conv1_2')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps2 = PrimaryCap2(conv2, dim_capsule=8, n_channels=32, kernel_size=(5, 5), strides=2, padding='same')
print "befor call <======="
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps2 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
name='digitcaps1_2')(primarycaps2)
print "after call <======="
poses_list.append(digitcaps2)
print("before pose[0]:", poses_list[0])
# print("convert_to_tensor:", tf.convert_to_tensor(poses_list))
# poses = tf.reduce_mean(tf.convert_to_tensor(poses_list), axis=0)
# print("reduce_mean type: ", type(poses))
# print("reduce_mean : ", poses)
# print("pose:", poses)
# print("pose[0]:", poses[0])
# pose.layers.average = tf.reduce_mean(poses_list, axis=0)
avg = layers.average(poses_list)
print("avg:", avg)
digitcaps = avg;
# digitcaps = K.sqrt(K.sum(K.square(avg), 2, True))
# digitcaps = K.sqrt(K.sum(K.square(avg), 2, True))
print("digitcaps type: ", type(digitcaps))
print("digitcaps: ", digitcaps)
# digitcaps = poses;
# print("digitcaps1 type: ", type(digitcaps1))
# print("digitcaps1: ", digitcaps1)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
print("out_caps: ", out_caps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16*n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def define_architecture(self, in_shape, hp_params):
# print('in_shape')
# print(in_shape)
# Verify
n_class = 1 # Binary classification
print(hp_params)
if hp_params == None:
p = self.default_space()
else:
p = hp_params
routings = p['routings']
conv01_filters = p['conv01_filters']
conv01_ksize = p['conv01_ksize']
conv02_ksize = p['conv02_ksize']
dim_capsule1 = p['dim_capsule1']
dim_capsule2 = p['dim_capsule2']
n_channels = p['n_channels']
in_shape = in_shape[1:]
maxlen = in_shape[0]
# Input
in_layer = Input(shape=(in_shape[0], in_shape[1], 1),
name='input_layer')
conv1 = Conv2D(filters=conv01_filters,
kernel_size=(4, conv01_ksize),
strides=1,
padding='valid',
activation='relu',
name='conv1')(in_layer)
conv1 = AveragePooling2D((1, 3))(conv1)
primarycaps = PrimaryCap(conv1,
dim_capsule=dim_capsule1,
n_channels=n_channels,
kernel_size=(1, conv02_ksize),
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=dim_capsule2,
routings=routings,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = Input(shape=(n_class, ), name='input_decoder')
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
# masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
x_recon = layers.Dense(512, activation='relu',
name='decode_dense_01')(masked_by_y)
x_recon = layers.Dense(1024, activation='relu',
name='decode_dense_02')(x_recon)
# # x_recon = layers.Dropout(.2, name='decode_drop_01')(x_recon)
x_recon = layers.Dense(np.prod(in_shape),
activation='sigmoid',
name='decode_dense_03')(x_recon)
x_recon = layers.Reshape(target_shape=in_shape,
name='out_recon')(x_recon)
# Shared Decoder model in training and prediction
# decoder = Sequential(name='decoder')
# decoder.add(Dense(512, activation='relu', input_dim=16*n_class))
# decoder.add(Dense(1024, activation='relu'))
# x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon)
# decoder.add(Dense(np.prod((in_shape[1], in_shape[2],1)), activation='sigmoid'))
# decoder.add(Reshape(target_shape=(in_shape[1], in_shape[2],1), name='out_recon'))
# train_model = Model([in_layer, y], [out_caps, decoder(masked_by_y)])
train_model = Model([in_layer, y], [out_caps, x_recon])
return train_model
def capsModel(input_shape, n_class, num_routing):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=512,
kernel_size=7,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
BN1 = layers.BatchNormalization()(conv1)
conv2 = layers.Conv2D(filters=256,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv2')(BN1)
BN2 = layers.BatchNormalization()(conv2)
conv3 = layers.Conv2D(filters=256,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv3')(BN2)
BN3 = layers.BatchNormalization()(conv3)
conv4 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv4')(BN3)
BN4 = layers.BatchNormalization()(conv4)
conv5 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv5')(BN4)
BN5 = layers.BatchNormalization()(conv5)
conv6 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv6')(BN5)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv6,
dim_capsule=24,
n_channels=32,
kernel_size=12,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def MultiLevelDCNet(input_shape, n_class, routings):
"""
A Multi-level DCNet on CIFAR-10.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
"""
x = layers.Input(shape=input_shape)
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
########################### Level 1 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(x, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 1)
# Here PrimaryCapsConv2D is the Conv2D output which is used as the primary capsules by reshaping and squashing (squash activation).
# primarycaps_1 (size: [None, num_capsule, dim_capsule]) is the "reshaped and sqashed output" which will be further passed to the dynamic routing protocol.
primarycaps_1, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid')
# Applying ReLU Activation to primary capsules
conv = layers.Activation('relu')(PrimaryCapsConv2D)
########################### Level 2 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 2)
primarycaps_2, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid')
# Applying ReLU Activation to primary capsules
conv = layers.Activation('relu')(PrimaryCapsConv2D)
########################### Level 3 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 3)
primarycaps_3, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=3, strides=2, padding='valid')
# Merging Primary Capsules for the Merged DigitCaps (CapsuleLayer formed by combining all levels of primary capsules)
mergedLayer = layers.merge([primarycaps_1,primarycaps_2,primarycaps_3], mode='concat', concat_axis=1)
########################### Separate DigitCaps Outputs (used for training) ###########################
# Merged DigitCaps
digitcaps_0 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
name='digitcaps0')(mergedLayer)
out_caps_0 = Length(name='capsnet_0')(digitcaps_0)
# First Level DigitCaps
digitcaps_1 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
name='digitcaps1')(primarycaps_1)
out_caps_1 = Length(name='capsnet_1')(digitcaps_1)
# Second Level DigitCaps
digitcaps_2 = CapsuleLayer(num_capsule=n_class, dim_capsule=12, routings=routings,
name='digitcaps2')(primarycaps_2)
out_caps_2 = Length(name='capsnet_2')(digitcaps_2)
# Third Level DigitCaps
digitcaps_3 = CapsuleLayer(num_capsule=n_class, dim_capsule=10, routings=routings,
name='digitcaps3')(primarycaps_3)
out_caps_3 = Length(name='capsnet_3')(digitcaps_3)
########################### Combined DigitCaps Output (used for evaluation) ###########################
digitcaps = layers.merge([digitcaps_1,digitcaps_2,digitcaps_3, digitcaps_0], mode='concat', concat_axis=2,
name='digitcaps')
out_caps = Length(name='capsnet')(digitcaps)
# Reconstruction (decoder) network
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(600, activation='relu', input_dim=int(digitcaps.shape[2]*n_class), name='zero_layer'))
decoder.add(layers.Dense(600, activation='relu', name='one_layer'))
decoderFinal = models.Sequential(name='decoderFinal')
# Concatenating two layers
decoderFinal.add(layers.Merge([decoder.get_layer('zero_layer'), decoder.get_layer('one_layer')], mode='concat'))
decoderFinal.add(layers.Dense(1200, activation='relu'))
decoderFinal.add(layers.Dense(np.prod([32,32,1]), activation='sigmoid'))
decoderFinal.add(layers.Reshape(target_shape=[32,32,1], name='out_recon'))
# Model for training
train_model = models.Model([x, y], [out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked_by_y)])
# Model for evaluation (prediction)
# Note that out_caps is the final prediction. Other predictions could be used for analysing separate-level predictions.
eval_model = models.Model(x, [out_caps, out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked)])
return train_model, eval_model
def CapsNet(gene, input_shape, n_class, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
inputs = layers.Input(shape=input_shape)
x = inputs
print("### Input shape", )
# if input_shape==(28, 28, 1):
# dataset="mnist"
# elif input_shape==(32, 32, 3):
# dataset="cifar"
# elif input_shape==(32, 32, 3) and gene[len(gene)-1][0]==2:
# dataset="cifar_resized"
remaining=len(gene)-2
conv_index=1
caps_index=1
deepcaps_index=1
count=0
xtra_skip=gene[len(gene)-2][0]
print("xtra_skip is " +str(gene[len(gene)-2][0])+"\n")
# Scan for type of layers
#N_layers = int((len(gene)-2)/9) # -1 for xtraskip term
conv=[]
caps=[]
dcaps=[]
N_conv=0
N_caps=0
N_dcaps=0
i=0
prevlayertype=0
for index in range(0,len(gene)):
if gene[index][0]==0:
N_conv = N_conv+1
conv.append(i)
i+=1
elif gene[index][0]==1:
type=1
N_caps = N_caps+1
caps.append(i)
i+=1
elif gene[index][0]==2:
convert=1
type=2
N_dcaps = N_dcaps+1
dcaps.append(i)
i+=1
for index in range(0,len(gene)):
#for layer in gene:
if len(gene[index])>1:
# Convolutional layers
# conv = [0, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules]
if gene[index][0]==0:
if prevlayertype==2:
x=layers.Reshape((gene[index][1], gene[index][1], -1))(x)
print("ConvertToConv"+str(prevlayertype))
x = layers.Conv2D(filters=gene[index][7]*gene[index][8], kernel_size=gene[index][4], strides=gene[index][5], padding='same', data_format="channels_last")(x)
x = BatchNormalization()(x)
print(x.get_shape().as_list())
conv_index=conv_index+1
remaining=remaining-1
print("Added Conv%s_layer" % str(conv_index-1))
count +=1
if xtra_skip==count-1 or (xtra_skip==0 and count==1):
x1 = x
xtra_skip=-2 #flag==2 for inserted xtra skip
prevlayertype=0
# Primary Capsules layers
# caps = [1, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules]
elif gene[index][0]==1 and remaining>1:
x = PrimaryCap(x, dim_capsule=gene[index][8], n_channels=gene[index][7], kernel_size=gene[index][4], strides=gene[index][5], padding='same') #CapsuleLayer(layer[7], layer[8])(x)
print(x.get_shape().as_list())
caps_index=caps_index+1
remaining=remaining-1
print("rem "+str(remaining))
print("Added Caps%s_layer" % str(caps_index-1))
veclen=gene[index][7]
numout=gene[index][8]
prevlayertype=1
# Class Capsules layer
# caps = [1, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules]
elif gene[index][0]==1 and remaining==1:
print(x.get_shape().as_list())
x = layers.Reshape(target_shape=[-1, gene[index][2]])(x) #[veclen*layer[1]*layer[1], numout])(x) # added this to solve error instead of Reshape(target_shape=[-1, layer[2]])(x)
print(x.get_shape().as_list())
x = CapsuleLayer(gene[index][7], gene[index][8], 3, name='digitcaps')(x)
print(x.get_shape().as_list())
dim_capsule_out=gene[index][8]
caps_index=caps_index+1
remaining=remaining-1
print(x.get_shape().as_list())
print("Added ClassCaps_layer\n")
# DeepCaps Cells
# d_caps = [2, insize, inchannels, incapsules, kernsize, stride, outsize, outchannels, outcapsules]
elif gene[index][0]==2 and remaining>1:
if prevlayertype==0:
x = ConvertToCaps()(x)
print("ConvertToCaps"+str(prevlayertype))
print(x.get_shape().as_list())
x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(gene[index][5], gene[index][5]), r_num=1, b_alphas=[1, 1, 1])(x)
deepcaps_index=deepcaps_index+1
if remaining==2:
x_skip = ConvCapsuleLayer3D(kernel_size=3, num_capsule=gene[index][7], num_atoms=gene[index][8], strides=1, padding='same', routings=3)(x)
deepcaps_index=deepcaps_index+1
else:
x_skip = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x)
deepcaps_index=deepcaps_index+1
x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x)
deepcaps_index=deepcaps_index+1
x = Conv2DCaps(gene[index][7], gene[index][8], kernel_size=(gene[index][4], gene[index][4]), strides=(1, 1), r_num=1, b_alphas=[1, 1, 1])(x)
x = layers.Add()([x, x_skip])
count +=1 #CapsCell count
print("Added CapsCell_layer\n")
if xtra_skip-1==count-1:
x1 = x
xtra_skip=-2 #flag==2 for inserted xtra skip
elif remaining==1 and xtra_skip==-2:
x2 = x
elif remaining==2 and xtra_skip==-1:
x1 = x
x2 = x
deepcaps_index=deepcaps_index+1
remaining=remaining-1
x2 = x
prevlayertype=2
#FlattenCaps
elif gene[index][0]==2 and remaining==1:
print(x1.get_shape().as_list())
print(x2.get_shape().as_list())
flatCaps = FlattenCaps()
xa = flatCaps(x2)
flatCaps = FlattenCaps()
xb = flatCaps(x1)
x = layers.Concatenate(axis=-2)([xa, xb])
print(xa.get_shape().as_list())
x = DClassCaps(num_capsule=gene[index][7], dim_capsule=gene[index][8], routings=3, channels=0, name='digit_caps')(x)
print("Added FlattenCaps_layer\n")
print(x.get_shape().as_list())
dim_capsule_out=gene[index][8]
else:
break
digitcaps = x
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
# DeepCaps Decoder
if input_shape == (32, 32, 3): # cifar10 resized
decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16
decoder.add(layers.Reshape((8, 8, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
#decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(3, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(32, 32, 3), name='out_recon')) # 64, 64 in origial code
elif input_shape == (64, 64, 3): # cifar 10 resized
decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16
decoder.add(layers.Reshape((8, 8, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(3, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(64, 64, 3), name='out_recon'))
elif input_shape == (28, 28, 1): # mnist
decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16
decoder.add(layers.Reshape((7, 7, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(1, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(28, 28, 1), name='out_recon'))
elif input_shape == (56, 56, 1): # mnist resized
decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16
decoder.add(layers.Reshape((7, 7, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(4, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(56, 56, 1), name='out_recon'))
elif input_shape == (56, 56, 3): # mnist resized
decoder.add(layers.Dense(7*7*16, input_dim=dim_capsule_out*n_class, activation="relu")) #7*7*16
decoder.add(layers.Reshape((7, 7, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(12, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.summary()
decoder.add(layers.Reshape(target_shape=(56, 56, 3), name='out_recon'))
elif input_shape == (64, 64, 1): # mnist resized for deepcaps (64x64 inputs)
decoder.add(layers.Dense(8*8*16, input_dim=dim_capsule_out*n_class, activation='relu')) #8*8*16
decoder.add(layers.Reshape((8, 8, 16)))
decoder.add(layers.BatchNormalization(momentum=0.8))
decoder.add(layers.Conv2DTranspose(64, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Conv2DTranspose(32, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(16, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(8, (3, 3), strides=(2, 2), padding="same"))
decoder.add(layers.Conv2DTranspose(1, (3, 3), strides=(1, 1), padding="same"))
decoder.add(layers.Activation("relu"))
decoder.add(layers.Reshape(target_shape=(64, 64, 1), name='out_recon'))
else:
raise NotImplementedError(f"Unknown decoder for shape {input_shape}")
decoder.summary()
# Models for training and evaluation (prediction)
train_model = models.Model([inputs, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(inputs, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, dim_capsule_out))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([inputs, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, num_routing):
from keras import layers, models
from capsulelayers import CapsuleLayer, PrimaryCap, Length, Mask
from keras.preprocessing import sequence
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 4d, [None, width, height, channels]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: A Keras Model with 2 inputs and 2 outputs
"""
x = layers.Input(shape=(maxlen, 1), dtype='float32')
# pool = layers.MaxPooling1D(pool_size=3, strides=1)(x)
# conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(embed)
conv1 = layers.Conv1D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
conv1 = layers.Dropout(0.1)(conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
# primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid')
primarycaps = PrimaryCap(conv1,
dim_vector=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
# digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps)
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_vector=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='out_caps')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def build_the_graph(self, params):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
self.import_nn_parameters(params)
start = time()
x = layers.Input(shape=self.input_shape)
# Layer 1: Just a conventional Conv2D layer
# params_conv_layer = self.params[0]
conv1 = layers.Conv2D(filters=self.conv_layer['filters'],
kernel_size=self.conv_layer['kernel_size'],
strides=self.conv_layer['strides'],
padding=self.conv_layer['padding'],
activation=self.conv_layer['activation'],
name=self.conv_layer['activation'])(x)
# filters=128,
# kernel_size=9,
# strides=1,
# padding='valid',
# activation='relu',
# name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(
conv1,
dim_capsule=self.primary_caps_layer['dim_capsule'],
n_channels=self.primary_caps_layer['n_channels'],
kernel_size=self.primary_caps_layer['kernel_size'],
strides=self.primary_caps_layer['strides'],
padding=self.primary_caps_layer['padding'])
# dim_capsule=8,
# n_channels=32,
# kernel_size=2,
# strides=2,
# padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(
num_capsule=self.n_class,
dim_capsule=self.digit_caps_layer['dim_capsule'],
# /dim_capsule = 16
routings=self.routings,
name=self.digit_caps_layer['name'])(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(self.n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(
digitcaps
) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(
layers.Dense(self.decoder_layer['first_dense'],
activation='relu',
input_dim=self.digit_caps_layer['dim_capsule'] *
self.n_class))
decoder.add(
layers.Dense(self.decoder_layer['second_dense'],
activation='relu'))
# decoder.add(layers.Dropout(0.5))
# decoder.add(layers.Dense(128, activation='relu', input_dim=16 * self.n_class))
# decoder.add(layers.Dense(256, activation='relu'))
# decoder.add(layers.Dense(np.prod(self.input_shape), activation='sigmoid'))
decoder.add(
layers.Dense(np.prod(self.input_shape), activation='softmax'))
decoder.add(
layers.Reshape(target_shape=self.input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(
shape=(self.n_class, self.digit_caps_layer['dim_capsule'])) # 16
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise],
decoder(masked_noised_y))
self.train_model = train_model
self.eval_model = eval_model
self.manipulate_model = manipulate_model
print('time to generate the model: {}'.format(time() - start))
return train_model, eval_model, manipulate_model
from keras import layers, models, optimizers
from keras.preprocessing.image import ImageDataGenerator
from keras import backend as K
n_class=2
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=128, kernel_size=3, strides=1, padding='valid', activation='relu', name='conv1')(x)
primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=3, kernel_size=9, strides=2, padding='valid')
CTscnaCaps = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,name='CTscnaCaps')(primarycaps)
out_caps = Length(name='capsnet')(CTscnaCaps)
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([CTscnaCaps, y])
masked = Mask()(CTscnaCaps)
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16*n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
noise = layers.Input(shape=(n_class, 16))
noised_CTscnaCaps = layers.Add()([CTscnaCaps, noise])
masked_noised_y = Mask()([noised_CTscnaCaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=num_classes,
dim_capsule=16,
num_routing=3,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(num_classes, ))
masked_by_y = Mask()([
digitcaps, y
]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(
digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * num_classes))
decoder.add(layers.Dropout(0.2))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
def CapsNet(input_shape, n_class, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv0 = layers.Conv2D(filters=64,
kernel_size=5,
strides=1,
padding='valid',
activation='relu',
name='conv0')(x)
#conv1 = layers.Conv2D(filters=128, kernel_size=5, strides=1, padding='valid', activation='relu', name='conv1')(conv0)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv0,
dim_capsule=8,
n_channels=32,
kernel_size=5,
strides=1,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
'''
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(256, activation='linear', input_dim=16*n_class))
decoder.add(layers.Dense(512, activation='linear'))
decoder.add(layers.Dense(1024, activation='linear'))
decoder.add(layers.Dense(np.prod(input_shape), activation='linear'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
decoder.summary()
'''
decoder = models.Sequential(name='decoder')
decoder.add(
layers.Dense(8 * 8 * 4, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Reshape((8, 8, 4)))
decoder.add(
layers.Conv2D(128,
kernel_size=5,
strides=1,
padding='same',
activation='relu'))
decoder.add(layers.UpSampling2D())
#16x16x128
decoder.add(
layers.Conv2D(64,
kernel_size=5,
strides=1,
padding='same',
activation='relu'))
decoder.add(layers.UpSampling2D())
#32x32x64
decoder.add(
layers.Conv2D(3,
kernel_size=5,
strides=1,
padding='same',
activation='tanh'))
decoder.summary()
#64x64x3
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def MultiLevelDCNet(input_shape, n_class, routings):
"""
A DCNet (1-level DCNet) on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
"""
x = layers.Input(shape=input_shape)
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
########################### Primary Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(x,
growth_rate=32,
nb_layers=8,
nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis,
epsilon=1.1e-5)(conv)
# Creating Primary Capsules
# Here PrimaryCapsConv2D is the Conv2D output which is used as the primary capsules by reshaping and squashing (squash activation).
# primarycaps_1 (size: [None, num_capsule, dim_capsule]) is the "reshaped and sqashed output" which will be further passed to the dynamic routing protocol.
primarycaps, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
########################### DigitCaps Output ###########################
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps0')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
# Reconstruction (decoder) network
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(
layers.Dense(512,
activation='relu',
input_dim=int(digitcaps.shape[2] * n_class),
name='zero_layer'))
decoder.add(layers.Dense(512, activation='relu', name='one_layer'))
decoderFinal = models.Sequential(name='decoderFinal')
# Concatenating two layers
decoderFinal.add(
layers.Merge(
[decoder.get_layer('zero_layer'),
decoder.get_layer('one_layer')],
mode='concat'))
decoderFinal.add(layers.Dense(1024, activation='relu'))
decoderFinal.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoderFinal.add(layers.Reshape(input_shape, name='out_recon'))
# Model for training
train_model = models.Model([x, y], [out_caps, decoderFinal(masked_by_y)])
# Model for evaluation (prediction)
eval_model = models.Model(x, [out_caps, decoderFinal(masked)])
return train_model, eval_model
def CapsNet_WithDecoder(input_shape, n_class, kernel, primary_channel,
primary_veclen, digit_veclen, dropout, routings,
decoderParm):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
if kernel > 19:
conv_padding = 'same'
print('kernel size big, padding to same')
else:
conv_padding = 'valid'
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=kernel,
strides=1,
padding=conv_padding,
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=primary_veclen,
n_channels=primary_channel,
kernel_size=kernel,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=digit_veclen,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
NumDecoderLayer, DecoderLayerUnit = decoderParm
decoder = models.Sequential(name='decoder')
decoder.add(
layers.Dense(DecoderLayerUnit[0],
activation='relu',
input_dim=digit_veclen * n_class)) #16*n_class))
for i in range(NumDecoderLayer - 2):
decoder.add(layers.Dense(DecoderLayerUnit[i + 1], activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, digit_veclen))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings):
"""
A Capsule Network on CIFAR-10.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the
second one for evaluation. `eval_model` can also be used for
training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(
filters=256,
kernel_size=9,
strides=1,
padding="valid",
activation="relu",
name="conv1",
)(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to
# [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=64,
kernel_size=9,
strides=2,
padding="valid")
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name="digitcaps")(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with
# its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name="capsnet")(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name="decoder")
decoder.add(layers.Dense(512, activation="relu", input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation="relu"))
decoder.add(layers.Dense(np.prod(input_shape), activation="sigmoid"))
decoder.add(layers.Reshape(target_shape=input_shape, name="out_recon"))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings):
"""
MNISTに関するカプセルネットワーク
:param input_shape: 入力データのshape(3次元で[width, height, channels]という形)
:param n_class: クラスの数
:param routings: routingを行う回数
:return 2つのmodel (1つ目:学習用モデル, 2つ目:評価用モデル)
`eval_model`というモデルも学習用としてしようすることもできる.
"""
x = layers.Input(shape=input_shape)
# 1層目: ただの2次元畳み込み層
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# 2層目: 活性化関数にsquash関数を用いた2次元畳み込み層で,[None ,num_capsule, dim_capsule]という形に変換する
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# 3層目: カプセル層 (routingアルゴリズムはここで行っている)
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# 4層目: ここはカプセルを"長さ"に変形するための補助レイヤーで, 教師データの形に合わせている.
# tensorflowを使用している場合, ここは必要ありません.
out_caps = Length(name='capsnet')(digitcaps)
# Decoderネットワーク
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y]) # 正解のラベルはよく学習のためにカプセル層の出力を隠す
masked = Mask()(digitcaps) # マスクは予測のために長さが最大値のカプセルを使用する
# このDecoderモデルは学習時にも予測時にも使われる.
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# 学習モデルと評価モデル
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# モデルにノイズを加える
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
print(input_shape)
print(n_class)
input_shape = (config.MAX_TEXT_LENGTH, )
n_class = 6
print(input_shape)
print(n_class)
x = layers.Input(shape=(config.MAX_TEXT_LENGTH, ))
x1 = layers.Embedding(
config.MAX_FEATURES,
config.embedding_dims,
input_length=config.MAX_TEXT_LENGTH,
# weights=[embedding_matrix],
trainable=True)(x)
x1 = layers.Dropout(0.2)(x1)
x1 = Reshape((config.MAX_TEXT_LENGTH, -1, 1))(x1)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# lr=layers.Dense(n_class, activation="sigmoid", name='softmax')(digitcaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# out_caps = layers.Dense(n_class, activation="sigmoid", name='softmax')(out_caps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
# train_model.summary()
# eval_model.summary()
# manipulate_model.summary()
return train_model, eval_model, manipulate_model