def CapsNet(input_shape, n_class, routings, model_version, lam_regularize):
"""
A Capsule Network .
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer, 对DEAP，kernel_size=9；对DREAMER，kernel_size=6
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1',
kernel_regularizer=regularizers.l2(lam_regularize))(
x) #kernel_size=9
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
# 对DEAP，kernel_size=9；对DREAMER，kernel_size=6
# 对CapsNet，strides=2，pading=‘valid’；对MLF-CapsNet，stides=1，padding='same'
if model_version == 'v0':
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid',
lam_regularize=lam_regularize,
model_version=model_version)
else:
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=1,
padding='same',
lam_regularize=lam_regularize,
model_version=model_version) #kernel_size=9
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps',
lam_regularize=lam_regularize)(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Models for training and evaluation (prediction)
y = layers.Input(shape=(n_class, ))
train_model = models.Model([x, y], out_caps)
eval_model = models.Model(x, out_caps)
return train_model, eval_model
def build_discriminator():
img = Input(shape=img_shape)
# x = UpSampling2D()(x)
# x = Conv2D(128, kernel_size=3, strides=1, padding="same")(img)
# x = LeakyReLU(alpha=0.2)(x)
# x = BatchNormalization(momentum=0.8)(x)
x = Conv2D(256, kernel_size=9, strides=1, padding="valid")(img)
x = LeakyReLU(alpha=0.2)(x)
x = BatchNormalization(momentum=0.8)(x)
x = PrimaryCap(x,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
x = CapsuleLayer(num_capsule=10, dim_capsule=32, routings=routings)(x)
# x = Mask()(x)
# y = layers.Input(shape=(num_classes+1,))
# x2 = Mask()([x, y])
# # x2 = Length(name='capsnet')(x)
# x1 = Mask()(x)
x = Flatten()(x)
# Determine validity and label of the image
validity = Dense(1, activation="sigmoid", name='ds')(x)
label = Dense(num_classes, activation="softmax", name='label')(x)
return Model(img, [validity, label])
def prepareModel():
"""
Capsule Network on biopsy patches.
"""
num_routing = 3
capsule_dimension = 8,
no_of_channels = 16
x = Input(shape=(224, 224, 3))
# Layer 1: Just a conventional Conv2D layer
conv1 = Conv2D(filters=32,
kernel_size=5,
strides=2,
padding='valid',
name='conv1')(x)
conv1 = Activation('relu', name='relu1')(conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=16,
kernel_size=5,
strides=2,
padding='valid')
primarycaps = Reshape((53 * 53, 16, 8))(primarycaps)
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = TimeDistributed(
CapsuleLayer(num_capsule=1,
dim_capsule=8,
num_routing=num_routing,
name='digitcaps'))(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
output = TrimmedAveragePool()(out_caps)
model = Model(inputs=x, outputs=output)
model.summary()
return model
def CapsNet(input_shape, n_class, routings):
x = Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = Conv2D(filters=128,
kernel_size=9,
strides=2,
padding='valid',
activation='relu',
name='conv1')(x)
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=16,
kernel_size=9,
strides=4,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
out_caps = Flatten()(digitcaps)
# outputs = Dense(278)(out_caps)
# Models for training and evaluation (prediction)
train_model = models.Model(x, out_caps)
return train_model
def CNN(seq_length, length, input_size, feature_maps, kernels, x):
concat_input = []
for feature_map, kernel in zip(feature_maps, kernels):
reduced_l = length - kernel + 1
#changes
# conv1 = Conv2D(feature_map, (1, kernel), activation='tanh', data_format="channels_last")(x)
conv1 = Conv2D(feature_map, (1, kernel),
strides=1,
activation='tanh',
data_format="channels_last")(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
primarycaps = PrimaryCap(conv1,
dim_vector=feature_map,
n_channels=32,
kernel_size=(1, kernel),
strides=2,
padding='valid',
filters=feature_map)
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=seq_length,
dim_vector=feature_map,
num_routing=3)(primarycaps)
# conv = Conv2D(feature_map, (1, kernel), activation='tanh', data_format="channels_last")(x)
# maxp = MaxPooling2D((1, reduced_l), data_format="channels_last")(conv)
concat_input.append(digitcaps)
x = Concatenate()(concat_input)
x = Reshape((seq_length, sum(feature_maps)))(x)
return x
def CapsNet(input_shape, n_class, num_routing):
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu')(x)
primarycaps = PrimaryCap(conv1,
dim_vector=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_vector=16,
num_routing=num_routing)(primarycaps)
out_caps = Length(name='out_caps')(digitcaps)
# Reconstruction network
y = layers.Input(shape=(n_class, ))
masked = Mask()([digitcaps, y])
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(784, activation='sigmoid')(x_recon)
x_recon = layers.Reshape(target_shape=[28, 28, 1],
name='out_recon')(x_recon)
return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, num_routing, kernel_size=7):
"""
A Capsule Network on 2019ncon.
:param input_shape: data shape, 3d, [width, height, channels],for example:[21,28,1]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: A Keras Model with 2 inputs and 2 outputs
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=128, kernel_size=kernel_size, strides=1, padding='same', activation='relu', name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
primarycaps = PrimaryCap(conv1, dim_vector=16, n_channels=32, kernel_size=kernel_size, strides=2, padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=32, num_routing=num_routing, name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='out_caps')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer.
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(np.prod(input_shape), activation='sigmoid')(x_recon)
x_recon = layers.Reshape(target_shape=input_shape, name='out_recon')(x_recon)
# two-input-two-output keras Model
return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, num_routing):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 4d, [None, width, height, channels]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: A Keras Model with 2 inputs and 2 outputs
"""
x = layers.Input(shape=(maxlen,))
embed = layers.Embedding(max_features, embed_dim, input_length=maxlen)(x)
conv1 = layers.Conv1D(filters=256, kernel_size=9, strides=1, padding='valid', activation='relu', name='conv1')(
embed)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_vector]
primarycaps = PrimaryCap(conv1, dim_vector=8, n_channels=32, kernel_size=9, strides=2, padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class, dim_vector=16, num_routing=num_routing, name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='out_caps')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer.
x_recon = layers.Dense(512, activation='relu')(masked)
x_recon = layers.Dense(1024, activation='relu')(x_recon)
x_recon = layers.Dense(maxlen, activation='sigmoid')(x_recon)
# two-input-two-output keras Model
return models.Model([x, y], [out_caps, x_recon])
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
embed = layers.Embedding(output_dim=emb_output_dim,
input_dim=length,
trainable=emb_train)(x)
conv1 = layers.Conv1D(filters=conv1_filt,
kernel_size=conv1_kern,
strides=conv1_stride,
padding=conv1_pad,
activation=conv1_act,
name='conv1')(embed)
primarycaps = PrimaryCap(conv1,
dim_capsule=caps1_dim,
n_channels=caps1_n_channels,
kernel_size=caps1_kern,
strides=caps1_stride,
padding=caps1_pad)
digitcaps = CapsuleLayer(num_capsule=caps2_num,
dim_capsule=caps2_dim,
routings=routings,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
train_model = models.Model(x, out_caps)
return train_model
def CapsNet(input_shape, n_class, routings, batch_size):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
out = models.Model(inputs=x, outputs=out_caps)
return out
def make_model():
x = Input(shape=input_shape)
conv1 = Conv3D(filters=conv_layer_filters, kernel_size=first_layer_kernel_size, strides=1,
padding='valid', activation='relu', name='conv1')(x)
primarycaps = PrimaryCap(conv1, dim_capsule=dim_primary_capsule, n_channels=n_channels,
kernel_size=primary_cap_kernel_size, strides=2, padding='valid',
name='primarycap_conv3d')
sub_caps = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_sub_capsule,
routings=3, name='sub_caps')(primarycaps)
out_caps = Length(name='capsnet')(sub_caps)
# Decoder network
y = Input(shape=(n_class,))
masked_by_y = Mask()([sub_caps, y])
masked = Mask()(sub_caps)
# shared decoder model in training and prediction
decoder = Sequential(name='decoder')
decoder.add(Dense(512, activation='relu',
input_dim=dim_sub_capsule*n_class))
decoder.add(Dense(1024, activation='relu'))
decoder.add(Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(Reshape(target_shape=input_shape, name='out_recon'))
### Models for training and evaluation (prediction and actually using)
train_model = Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = Model(x, [out_caps, decoder(masked)])
### manipulate model can be used to visualize activation maps for specific classes
noise = Input(shape=(n_class, dim_sub_capsule))
noised_sub_caps = Add()([sub_caps, noise])
masked_noised_y = Mask()([noised_sub_caps, y])
manipulate_model = Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, num_routing):
"""
MNIST Veriseti için Kapsül Ağları.
:param input_shape: veri şekli, 3d, [genişlik, yükseklik, kanal]
:param n_class: sınıf sayısı
:param num_routing: dinamik yönlendirme (dynamic routing) iterasyon sayısı
:return: iki Keras model, birincisi eğitim için, ikincisi değerlendirme için (evalaution).
`eval_model` aynı zamanda eğitim için de kullanılabilir.
"""
x = layers.Input(shape=input_shape)
# Katman 1: Geleneksel Evrişimli Sinir Ağı Katmanı (Conv2D)
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Katman 2: Conv2D katmanı ile ezme (squash) aktivasyonu, [None, num_capsule, dim_capsule]’e yeniden şekil veriliyor.
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Katman 3: Kapsül Katmanı. Dinamik Yönlendirme algoritması burada çalışıyor.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Katman 4: Her kapsülün uzunluğunu yeniden düzenleyen yardımcı bir katmandır. Doğru etiketle eşleşmesi için bu işlem yapılır.
# Eğer Tensorflow kullanıyorsanız bu işleme gerek yoktur. :)
out_caps = Length(name='capsnet')(digitcaps)
# Kodçözücü Ağ.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # Doğru etiket, kapsül katmanın çıkışını maskelemek için kullanılır (Eğitim için).
masked = Mask()(
digitcaps
) # Filtre (maske), kapsülün maksimal uzunluğu ile kullanılır (Kestirim için).
# Eğitim ve Kestirimde Kodçözücü Modelin Paylaşımı
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Eğitim ve Değerlendirme (Kestirim) için Modeller
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
"""
A Regular Layer: Tensor with shape [N, H, W, C], where often
N is the number of samples e.g., batch_size,
H is the height,
W is the width, and
C is the number of channels.
For example, in MNIST, each image is 28x28, with a single channel, so a batch size of 128 images would lead to an input tensor of [128, 28, 28, 1].
A Regular Kernel: Tensor with shape [KH, KW, I, O], where often
KH is the kernel height,
KW is the kernel width,
I is the number of input channels, and
O is the number of output channels,
e.g., a 5x5 convolution kernel on the previous [128, 28, 28, 1] to provide 32 ouptut channels would require a kernel shape [5, 5, 1, 32]. Of course, we also need strides to determine the height and width of the output layer.
A Matrix Capsule Layer: Tensor tuple of (poses: [N, H, W, C, PH, PW], activations: [N, H, W, C]), where
PH is the pose height, and
PW is the pose width.
This is an extension from the regular layers to include poses and activations in representing a feature or an object. In the paper, matrix capsules with EM routing, PH = PW = 4.
A Matrix Capsule Kernel: Tensor with shape [KH, KW, I, O, PH, PW].
This kernel is used in the matrix capsules convolution operation to convert an inputs matrix capsule layer (poses: [N, H, W, I, PH, PW], activations: [N, H, W, I]) into and
output matrix capsule layer (poses: [N, OH, OW, O, PH, PW], activations: [N, OH, OW, O]). Here, OH is the output height, OW is the output width, and OH and OW are determined by the KH, KW and strides.
In addition, I assume we use the MNIST dataset with a batchsize N=128, set A=32, B=48, C=64, D=80, and _E=10, so that we can distinguish between each layers.
"""
# Layer 1: Just a conventional Conv2D layer
# Input Layer -> AL (A=32): kernel [5, 5, 1, 32], strides [1, 2, 2, 1], padding SAME, ReLU. This is a regular convoluation operation connects IL to AL.
# inputs [N, H, W, C] -> conv2d, 5x5, strides 2, channels 32 -> nets [N, OH, OW, 32]
conv1 = layers.Conv2D(filters=32, kernel_size=5, strides=2, padding='same', activation='relu', name='conv1')(x)
# Layer AL: [128, 14, 14, 32].
# Layer 2: Capsule Initialize
# AL -> BL (B=48): kernel [1, 1, 32, 48] x (4 x 4 + 1), strides [1, 1, 1, 1].
# 16 such kernels of [1, 1, 32, 48] for building poses, and 1 such kernel [1, 1, 32, 48] for building activation. This is the initialization operation to connect
# a regular layer to a matrix capsule layer, implemented in capsule_init() with details in next section.
# inputs [N, H, W, C] -> conv2d, 1x1, strides 1, channels 32x(4x4+1) -> (poses, activations)
poses, activations = PrimaryCap(conv1, matrix_dims_in_capsule=4, n_channels=32, kernel_size=1, strides=1, padding='valid')
# Layer 3: Capsule Conv 1
# inputs: (poses, activations) -> capsule-conv 3x3x32x32x4x4, strides 2 -> (poses, activations)
nets = capsules_conv(
nets, shape=[3, 3, 32, 32], strides=[1, 2, 2, 1], iterations=iterations, name='capsule_conv1'
)
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu',
input_dim=16 * n_class)) # YES
decoder.add(layers.Dense(1024, activation='relu')) # YES
decoder.add(layers.Dense(4096, activation='relu')) # YES
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, num_routing):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param num_routing: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
other_masks = []
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
# Layers 1: Convolution Layers to extract low level features
conv1 = layers.Conv2D(filters=64,
kernel_size=3,
strides=2,
activation='relu',
name='Conv1')(x)
conv2 = layers.Conv2D(filters=128,
kernel_size=3,
strides=2,
activation='relu',
name='Conv2')(conv1)
conv3 = layers.Conv2D(filters=256,
kernel_size=6,
strides=2,
activation='relu',
name='Conv3')(conv2)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv3,
dim_capsule=8,
n_channels=32,
kernel_size=6,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
classcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='classcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
out_caps = Length(name='capsnet')(classcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([classcaps, y])
masked = Mask()(classcaps)
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet(input_shape, n_class, routings):
"""
针对 MNIST 手写字符识别的胶囊网络
## input_shape: 输入数据的维度, 长度为3的列表, [width, height, channels]
## n_class: 类别的数量
## routings: routing算法迭代的次数
:return: 返回两个模型, 第一个用于训练, 第二个用于测试
"""
# Encoder network.输入为图片
x = layers.Input(shape=input_shape)
# Layer 1: 卷积层
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# Layer 2: PrimaryCap（自定义层）
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: CapsuleLayer（自定义胶囊层）
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: 模值计算层（自定义层）
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.输入为向量
y = layers.Input(shape=(n_class, ))
# 蒙版操作，对decoder输入的标准化
masked_by_y = Mask()([digitcaps, y]) # 用真实值取代胶囊层的输出值。此项用于训练
masked = Mask()(digitcaps) # 用最大长度值做胶囊的蒙版。此项用于评估
# 包含3层的全连接神经网络（序贯模型）
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# 最终将输出转为图片
# 训练模型和评估模型
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def build(self):
query = Input(name='query', shape=(self.config['text1_maxlen'],))
show_layer_info('Input', query)
doc = Input(name='doc', shape=(self.config['text2_maxlen'],))
show_layer_info('Input', doc)
dpool_index = Input(name='dpool_index', shape=[self.config['text1_maxlen'], self.config['text2_maxlen'], 3], dtype='int32')
show_layer_info('Input', dpool_index)
embedding = Embedding(self.config['vocab_size'], self.config['embed_size'], weights=[self.config['embed']], trainable = self.embed_trainable)
q_embed = embedding(query)
show_layer_info('Embedding', q_embed)
d_embed = embedding(doc)
show_layer_info('Embedding', d_embed)
cross = Dot(axes=[2, 2], normalize=False)([q_embed, d_embed])
show_layer_info('Dot', cross)
cross_reshape = Reshape((self.config['text1_maxlen'], self.config['text2_maxlen'], 1))(cross)
show_layer_info('Reshape', cross_reshape)
conv2d = Conv2D(self.config['kernel_count'], self.config['kernel_size'], padding='same', activation='relu')
dpool = DynamicMaxPooling(self.config['dpool_size'][0], self.config['dpool_size'][1])
conv1 = conv2d(cross_reshape)
show_layer_info('Conv2D', conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1, dim_capsule=8, n_channels=32, kernel_size=9, strides=2, padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=2, dim_capsule=16, routings=3,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
pool1 = dpool([conv1, dpool_index])
show_layer_info('DynamicMaxPooling', pool1)
pool1_flat = Flatten()(pool1)
show_layer_info('Flatten', pool1_flat)
pool1_flat_drop = Dropout(rate=self.config['dropout_rate'])(pool1_flat)
show_layer_info('Dropout', pool1_flat_drop)
if self.config['target_mode'] == 'classification':
out_ = Dense(2, activation='softmax')(pool1_flat_drop)
elif self.config['target_mode'] in ['regression', 'ranking']:
out_ = Dense(1)(pool1_flat_drop)
show_layer_info('Dense', out_)
model = Model(inputs=[query, doc, dpool_index], outputs=out_)
model.summary()
return model
def CapsNet_NoDecoder(input_shape, n_class, kernel, primary_channel,
primary_veclen, digit_veclen, dropout, routings,
model_size_info):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
model_size_info: kernel, conv_channel, primary_channel, input_veclen, output_veclen
"""
kernel, conv_channel, primary_channel, primary_veclen, digit_veclen = model_size_info
if kernel > 19:
conv_padding = 'same'
print('kernel size big, padding to same')
else:
conv_padding = 'valid'
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=kernel,
strides=1,
padding=conv_padding,
activation='relu',
name='conv1')(x)
if dropout != 0:
conv1 = layers.Dropout(dropout)(conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=primary_veclen,
n_channels=primary_channel,
kernel_size=kernel,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=digit_veclen,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Models for training and evaluation (prediction)
train_model = models.Model(x, out_caps)
return train_model
def build_model(input_shape):
# encoder
x = Input(shape=input_shape)
conv1 = Conv2D(filters=64,
kernel_size=3,
padding='same',
trainable=True,
name='conv1')(x)
conv1 = BatchNormalization()(conv1)
conv1 = Activation('relu')(conv1)
conv1 = Conv2D(filters=64,
kernel_size=3,
padding='same',
trainable=True,
name='conv2')(conv1)
conv1 = BatchNormalization()(conv1)
conv1 = Activation('relu')(conv1)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=N_CLASS,
dim_capsule=16,
routings=ROUTINGS,
name='digitcaps')(primarycaps)
digitcaps = Flatten()(digitcaps)
decoder = Dense(512, activation='relu', input_dim=16 * N_CLASS)(digitcaps)
decoder = Dense(1024, activation='relu')(decoder)
decoder = Dense(np.prod(input_shape), activation='sigmoid')(decoder)
decoder = Reshape(input_shape)(decoder)
model = Model(x, decoder)
# transfer weights from first 2 layers of VGG-19
weights = pre_trained_model.layers[1].get_weights()[0][:, :, :, :]
#weights = np.reshape(weights, (3,3,1,64))
bias = pre_trained_model.layers[1].get_weights()[1]
model.layers[1].set_weights([weights, bias])
weights = pre_trained_model.layers[2].get_weights()[0]
bias = pre_trained_model.layers[2].get_weights()[1]
model.layers[4].set_weights([weights, bias])
return model
def CapsNet(input_shape, n_class, routings):
left_input = layers.Input(shape=input_shape)
right_input = layers.Input(shape=input_shape)
input = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(input)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
tunnel_l = models.Model(input, out_caps)
tunnel_r = models.Model(input, out_caps)
encoded_l = tunnel_l(left_input)
encoded_r = tunnel_r(right_input)
L1_layer = layers.Lambda(lambda tensors: K.abs(tensors[0] - tensors[1]))
L1_distance = L1_layer([encoded_l, encoded_r])
prediction = layers.Dense(1, activation='sigmoid')(L1_distance)
train_model = models.Model(inputs=[left_input, right_input],
outputs=prediction)
return train_model
def CapsNet(input_shape, n_class, routings):
"""
A Capsule Network on MNIST.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
`eval_model` can also be used for training.
"""
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(
filters=256,
kernel_size=9,
strides=4, # TODO: use 5
padding='valid',
activation='relu',
name='conv1')(
x)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(
conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(
num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(
primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Models for training and evaluation (prediction)
model = models.Model(x, out_caps)
return model
def CapsNet(input_shape, n_class, routings):
x = layers.Input(shape=input_shape)
conv1 = layers.Conv2D(filters=256,
kernel_size=(9),
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
out_caps = Length(name='capsnet')(digitcaps)
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y])
masked = Mask()(digitcaps)
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# manipulate model
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def CapsNet(input_shape, n_class, routings, batch_size):
# input layer
x = tf.keras.layers.Input(input_shape, batch_size = batch_size)
# layer 1 : regular Convolutionnal layer
conv1 = tf.keras.layers.Conv2D(filters = 256, kernel_size = (9,9), activation = 'relu', name = 'conv1')(x)
# layer 2 : PrimaryCaps, which is a convolution layer with Squash activation
# dim_capsule : corresponds to the dimension of the capsule output vector
# n_channels : number of capsule types
primarycaps = PrimaryCap(conv1, dim_capsule = 8, n_channels = 32, kernel_size = 9, strides = 2, padding = 'valid')
# layer 3 : CapsuleLayer (involves routing by agreement)
# each capsule in this layer represents one of the Kuzushiji symbol
kcaps = CapsuleLayer(num_capsule = n_class, dim_capsule = 16, routings = routings, name = 'kcaps')(primarycaps)
# layer 4 : layer that takes the length of each capsule
out_caps = Length(name='capsnet')(kcaps)
# Let's build the decoder network
# 2 reconstructions are performed :
# - first one is to reconstruct image according to the true label
# - second one is to reconstruct image according to the vector with maximal length (prediction)
y = tf.keras.layers.Input((n_class,))
masked_by_y = Mask()([kcaps, y])
masked = Mask()(kcaps)
# Dense layers of the decoder architecture as described in the paper
decoder = tf.keras.models.Sequential(name = 'decoder')
decoder.add(tf.keras.layers.Dense(512, activation = 'relu', input_dim = 16 * n_class))
decoder.add(tf.keras.layers.Dense(1024, activation = 'relu'))
decoder.add(tf.keras.layers.Dense(input_shape[0]*input_shape[1], activation = 'sigmoid'))
decoder.add(tf.keras.layers.Reshape(input_shape, name = 'out_recon'))
# Models used for training and evaluation
# train_model involves training of the decoder
# while evaluation model, given an input x, outputs his prediction and his reconstruction using the trained decoder
train_model = tf.keras.models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = tf.keras.models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model
def CapsNet_nogradientstop_crossentropy(input_shape, n_class, routings): # best testing results! val 0.13xx testX cnn1 200 1 cnn2 150 9 drop1 0.68 drop20.68 n_channels 50 kernel_size 20,dropout1
x = layers.Input(shape=input_shape)
conv1 = layers.Conv1D(filters=200, kernel_size=1, strides=1, padding='valid', kernel_initializer='he_normal',activation='relu', name='conv1')(x)
#conv1=BatchNormalization()(conv1)
conv1 = Dropout(0.7)(conv1)
conv2 = layers.Conv1D(filters=200, kernel_size=9, strides=1, padding='valid', kernel_initializer='he_normal',activation='relu', name='conv2')(conv1)
#conv1=BatchNormalization()(conv1)
conv2 = Dropout(0.75)(conv2) #0.75 valx loss has 0.1278!
primarycaps = PrimaryCap(conv2, dim_capsule=8, n_channels=60, kernel_size=20, kernel_initializer='he_normal',strides=1, padding='valid',dropout=0.2)
dim_capsule_dim2=10
# Capsule layer. Routing algorithm works here.
digitcaps_c = CapsuleLayer_nogradient_stop(num_capsule=n_class, dim_capsule=dim_capsule_dim2, num_routing=routings,name='digitcaps',kernel_initializer='he_normal',dropout=0.1)(primarycaps)
#digitcaps_c = CapsuleLayer(num_capsule=n_class, dim_capsule=dim_capsule_dim2, num_routing=routings,name='digitcaps',kernel_initializer='he_normal')(primarycaps)
digitcaps = Extract_outputs(dim_capsule_dim2)(digitcaps_c)
weight_c = Extract_weight_c(dim_capsule_dim2)(digitcaps_c)
out_caps = Length()(digitcaps)
out_caps = Activation('softmax',name='capsnet')(out_caps)
# Decoder network.
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=dim_capsule_dim2*n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = Model(x, [out_caps, decoder(masked)])
weight_c_model = Model(x,weight_c)
# manipulate model
noise = layers.Input(shape=(n_class, dim_capsule_dim2))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model,weight_c_model
def create_layers(input_img, n_class, routings):
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=128,
kernel_size=5,
strides=1,
padding='valid',
activation='relu')(input_img)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=12,
kernel_size=5,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=4,
routings=routings)(primarycaps)
return digitcaps
def create_layers(input_img, n_class, routings):
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=256,
kernel_size=5,
strides=1,
padding='valid',
activation='relu')(input_img)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=5,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings)(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
return digitcaps
def CapsNet(input_shape, n_class, routings):
"""
MNISTに関するカプセルネットワーク
:param input_shape: 入力データのshape(3次元で[width, height, channels]という形)
:param n_class: クラスの数
:param routings: routingを行う回数
:return 2つのmodel (1つ目:学習用モデル, 2つ目:評価用モデル)
`eval_model`というモデルも学習用としてしようすることもできる.
"""
x = layers.Input(shape=input_shape)
# 1層目: ただの2次元畳み込み層
conv1 = layers.Conv2D(filters=256,
kernel_size=9,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
# 2層目: 活性化関数にsquash関数を用いた2次元畳み込み層で,[None ,num_capsule, dim_capsule]という形に変換する
primarycaps = PrimaryCap(conv1,
dim_capsule=8,
n_channels=32,
kernel_size=9,
strides=2,
padding='valid')
# 3層目: カプセル層 (routingアルゴリズムはここで行っている)
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
routings=routings,
name='digitcaps')(primarycaps)
# 4層目: ここはカプセルを"長さ"に変形するための補助レイヤーで, 教師データの形に合わせている.
# tensorflowを使用している場合, ここは必要ありません.
out_caps = Length(name='capsnet')(digitcaps)
# Decoderネットワーク
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()([digitcaps, y]) # 正解のラベルはよく学習のためにカプセル層の出力を隠す
masked = Mask()(digitcaps) # マスクは予測のために長さが最大値のカプセルを使用する
# このDecoderモデルは学習時にも予測時にも使われる.
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# 学習モデルと評価モデル
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
# モデルにノイズを加える
noise = layers.Input(shape=(n_class, 16))
noised_digitcaps = layers.Add()([digitcaps, noise])
masked_noised_y = Mask()([noised_digitcaps, y])
manipulate_model = models.Model([x, y, noise], decoder(masked_noised_y))
return train_model, eval_model, manipulate_model
def MultiLevelDCNet(input_shape, n_class, routings):
"""
A Multi-level DCNet on CIFAR-10.
:param input_shape: data shape, 3d, [width, height, channels]
:param n_class: number of classes
:param routings: number of routing iterations
:return: Two Keras Models, the first one used for training, and the second one for evaluation.
"""
x = layers.Input(shape=input_shape)
concat_axis = 1 if K.image_data_format() == 'channels_first' else -1
########################### Level 1 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(x, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 1)
# Here PrimaryCapsConv2D is the Conv2D output which is used as the primary capsules by reshaping and squashing (squash activation).
# primarycaps_1 (size: [None, num_capsule, dim_capsule]) is the "reshaped and sqashed output" which will be further passed to the dynamic routing protocol.
primarycaps_1, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid')
# Applying ReLU Activation to primary capsules
conv = layers.Activation('relu')(PrimaryCapsConv2D)
########################### Level 2 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 2)
primarycaps_2, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=5, strides=2, padding='valid')
# Applying ReLU Activation to primary capsules
conv = layers.Activation('relu')(PrimaryCapsConv2D)
########################### Level 3 Capsules ###########################
# Incorporating DenseNets - Creating a dense block with 8 layers having 32 filters and 32 growth rate.
conv, nb_filter = densenet.DenseBlock(conv, growth_rate=32, nb_layers=8, nb_filter=32)
# Batch Normalization
DenseBlockOutput = BatchNormalization(axis=concat_axis, epsilon=1.1e-5)(conv)
# Creating Primary Capsules (Level 3)
primarycaps_3, PrimaryCapsConv2D = PrimaryCap(DenseBlockOutput, dim_capsule=8, n_channels=12, kernel_size=3, strides=2, padding='valid')
# Merging Primary Capsules for the Merged DigitCaps (CapsuleLayer formed by combining all levels of primary capsules)
mergedLayer = layers.merge([primarycaps_1,primarycaps_2,primarycaps_3], mode='concat', concat_axis=1)
########################### Separate DigitCaps Outputs (used for training) ###########################
# Merged DigitCaps
digitcaps_0 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
name='digitcaps0')(mergedLayer)
out_caps_0 = Length(name='capsnet_0')(digitcaps_0)
# First Level DigitCaps
digitcaps_1 = CapsuleLayer(num_capsule=n_class, dim_capsule=16, routings=routings,
name='digitcaps1')(primarycaps_1)
out_caps_1 = Length(name='capsnet_1')(digitcaps_1)
# Second Level DigitCaps
digitcaps_2 = CapsuleLayer(num_capsule=n_class, dim_capsule=12, routings=routings,
name='digitcaps2')(primarycaps_2)
out_caps_2 = Length(name='capsnet_2')(digitcaps_2)
# Third Level DigitCaps
digitcaps_3 = CapsuleLayer(num_capsule=n_class, dim_capsule=10, routings=routings,
name='digitcaps3')(primarycaps_3)
out_caps_3 = Length(name='capsnet_3')(digitcaps_3)
########################### Combined DigitCaps Output (used for evaluation) ###########################
digitcaps = layers.merge([digitcaps_1,digitcaps_2,digitcaps_3, digitcaps_0], mode='concat', concat_axis=2,
name='digitcaps')
out_caps = Length(name='capsnet')(digitcaps)
# Reconstruction (decoder) network
y = layers.Input(shape=(n_class,))
masked_by_y = Mask()([digitcaps, y]) # The true label is used to mask the output of capsule layer. For training
masked = Mask()(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(600, activation='relu', input_dim=int(digitcaps.shape[2]*n_class), name='zero_layer'))
decoder.add(layers.Dense(600, activation='relu', name='one_layer'))
decoderFinal = models.Sequential(name='decoderFinal')
# Concatenating two layers
decoderFinal.add(layers.Merge([decoder.get_layer('zero_layer'), decoder.get_layer('one_layer')], mode='concat'))
decoderFinal.add(layers.Dense(1200, activation='relu'))
decoderFinal.add(layers.Dense(np.prod([32,32,1]), activation='sigmoid'))
decoderFinal.add(layers.Reshape(target_shape=[32,32,1], name='out_recon'))
# Model for training
train_model = models.Model([x, y], [out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked_by_y)])
# Model for evaluation (prediction)
# Note that out_caps is the final prediction. Other predictions could be used for analysing separate-level predictions.
eval_model = models.Model(x, [out_caps, out_caps_0, out_caps_1, out_caps_2, out_caps_3, decoderFinal(masked)])
return train_model, eval_model
def capsModel(input_shape, n_class, num_routing):
x = layers.Input(shape=input_shape)
# Layer 1: Just a conventional Conv2D layer
conv1 = layers.Conv2D(filters=512,
kernel_size=7,
strides=1,
padding='valid',
activation='relu',
name='conv1')(x)
BN1 = layers.BatchNormalization()(conv1)
conv2 = layers.Conv2D(filters=256,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv2')(BN1)
BN2 = layers.BatchNormalization()(conv2)
conv3 = layers.Conv2D(filters=256,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv3')(BN2)
BN3 = layers.BatchNormalization()(conv3)
conv4 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv4')(BN3)
BN4 = layers.BatchNormalization()(conv4)
conv5 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv5')(BN4)
BN5 = layers.BatchNormalization()(conv5)
conv6 = layers.Conv2D(filters=128,
kernel_size=3,
strides=1,
padding='valid',
activation='relu',
name='conv6')(BN5)
# Layer 2: Conv2D layer with `squash` activation, then reshape to [None, num_capsule, dim_capsule]
primarycaps = PrimaryCap(conv6,
dim_capsule=24,
n_channels=32,
kernel_size=12,
strides=2,
padding='valid')
# Layer 3: Capsule layer. Routing algorithm works here.
digitcaps = CapsuleLayer(num_capsule=n_class,
dim_capsule=16,
num_routing=num_routing,
name='digitcaps')(primarycaps)
# Layer 4: This is an auxiliary layer to replace each capsule with its length. Just to match the true label's shape.
# If using tensorflow, this will not be necessary. :)
out_caps = Length(name='capsnet')(digitcaps)
# Decoder network.
y = layers.Input(shape=(n_class, ))
masked_by_y = Mask()(
[digitcaps, y]
) # The true label is used to mask the output of capsule layer. For training
masked = Mask(
)(digitcaps) # Mask using the capsule with maximal length. For prediction
# Shared Decoder model in training and prediction
decoder = models.Sequential(name='decoder')
decoder.add(layers.Dense(512, activation='relu', input_dim=16 * n_class))
decoder.add(layers.Dense(1024, activation='relu'))
decoder.add(layers.Dense(np.prod(input_shape), activation='sigmoid'))
decoder.add(layers.Reshape(target_shape=input_shape, name='out_recon'))
# Models for training and evaluation (prediction)
train_model = models.Model([x, y], [out_caps, decoder(masked_by_y)])
eval_model = models.Model(x, [out_caps, decoder(masked)])
return train_model, eval_model