def configureLossLayers():
topology = training.Topology()
# pooling(loss1/ave_pool): 5x5 + 3x3s
loss_ave_pool = average_pooling2d.Batch(4)
loss_ave_pool.parameter.kernelSizes = pooling2d.KernelSizes(5, 5)
loss_ave_pool.parameter.strides = pooling2d.Strides(3, 3)
loss_ave_pool.parameter.paddings = pooling2d.Paddings(0, 0)
loss_ave_pool_id = topology.add(loss_ave_pool)
# convolution(loss/conv): 1x1@128 + 1x1s
loss_conv = convolution2d.Batch()
loss_conv.parameter.nKernels = 128
loss_conv.parameter.kernelSizes = convolution2d.KernelSizes(1, 1)
loss_conv.parameter.strides = convolution2d.Strides(1, 1)
loss_conv.parameter.paddings = convolution2d.Paddings(0, 0)
loss_conv.parameter.weightsInitializer = xavier.Batch()
loss_conv.parameter.biasesInitializer = uniform.Batch(0.2, 0.2)
loss_conv_id = topology.add(loss_conv)
# relu(loss/relu_conv)
loss_relu_conv = relu.Batch()
loss_relu_conv_id = topology.add(loss_relu_conv)
# fullyconnected(loss/fc): n = 1024
loss_fc = fullyconnected.Batch(1024)
loss_fc.parameter.weightsInitializer = xavier.Batch()
loss_fc.parameter.biasesInitializer = uniform.Batch(0.2, 0.2)
loss_fc_id = topology.add(loss_fc)
# relu(loss/relu_fc)
loss_relu_fc = relu.Batch()
loss_relu_fc_id = topology.add(loss_relu_fc)
# dropout(loss/drop_fc): p = 0.7
loss_drop_fc = dropout.Batch()
loss_drop_fc.parameter.retainRatio = 0.7
loss_drop_fc_id = topology.add(loss_drop_fc)
# fullyconnected(loss/classifier): n = 1000
loss_classifier = fullyconnected.Batch(1000)
loss_classifier.parameter.weightsInitializer = xavier.Batch()
loss_classifier.parameter.biasesInitializer = uniform.Batch(0.0, 0.0)
loss_classifier_id = topology.add(loss_classifier)
# softmax + crossentropy loss(loss/loss)
loss_loss = loss.softmax_cross.Batch()
loss_loss_id = topology.add(loss_loss)
topology.get(loss_ave_pool_id).addNext(loss_conv_id)
topology.get(loss_conv_id).addNext(loss_relu_conv_id)
topology.get(loss_relu_conv_id).addNext(loss_fc_id)
topology.get(loss_fc_id).addNext(loss_relu_fc_id)
topology.get(loss_relu_fc_id).addNext(loss_drop_fc_id)
topology.get(loss_drop_fc_id).addNext(loss_classifier_id)
topology.get(loss_classifier_id).addNext(loss_loss_id)
return topology
def build(node, trainable):
fc_layer = fullyconnected.Batch(node.num_biases())
fc_layer = add_initializer(node, fc_layer)
return build(node, trainable, fc_layer)
def configureNet():
# convolution: 11x11@96 + 4x4s
convolution1 = convolution2d.Batch()
convolution1.parameter.kernelSizes = convolution2d.KernelSizes(11, 11)
convolution1.parameter.strides = convolution2d.Strides(4, 4)
convolution1.parameter.nKernels = 96
convolution1.parameter.weightsInitializer = gaussian.Batch(0, 0.01)
convolution1.parameter.biasesInitializer = uniform.Batch(0, 0)
# relu
relu1 = relu.Batch()
# lrn: alpha=0.0001, beta=0.75, local_size=5
lrn1 = lrn.Batch()
lrn1.parameter.kappa = 1
lrn1.parameter.nAdjust = 5
lrn1.parameter.alpha = 0.0001 / lrn1.parameter.nAdjust
lrn1.parameter.beta = 0.75
# pooling: 3x3 + 2x2s
maxpooling1 = maximum_pooling2d.Batch(4)
maxpooling1.parameter.kernelSizes = pooling2d.KernelSizes(3, 3)
maxpooling1.parameter.paddings = pooling2d.Paddings(0, 0)
maxpooling1.parameter.strides = pooling2d.Strides(2, 2)
# convolution: 5x5@256 + 1x1s
convolution2 = convolution2d.Batch()
convolution2.parameter.kernelSizes = convolution2d.KernelSizes(5, 5)
convolution2.parameter.strides = convolution2d.Strides(1, 1)
convolution2.parameter.paddings = convolution2d.Paddings(2, 2)
convolution2.parameter.nKernels = 256
convolution2.parameter.nGroups = 2
convolution2.parameter.weightsInitializer = gaussian.Batch(0, 0.01)
convolution2.parameter.biasesInitializer = uniform.Batch(0, 0)
# relu
relu2 = relu.Batch()
# lrn: alpha=0.0001, beta=0.75, local_size=5
lrn2 = lrn.Batch()
lrn2.parameter.kappa = 1
lrn2.parameter.nAdjust = 5
lrn2.parameter.alpha = 0.0001 / lrn2.parameter.nAdjust
lrn2.parameter.beta = 0.75
# pooling: 3x3 + 2x2s
maxpooling2 = maximum_pooling2d.Batch(4)
maxpooling2.parameter.kernelSizes = pooling2d.KernelSizes(3, 3)
maxpooling2.parameter.paddings = pooling2d.Paddings(0, 0)
maxpooling2.parameter.strides = pooling2d.Strides(2, 2)
# convolution: 3x3@384 + 2x2s
convolution3 = convolution2d.Batch()
convolution3.parameter.kernelSizes = convolution2d.KernelSizes(3, 3)
convolution3.parameter.paddings = convolution2d.Paddings(1, 1)
convolution3.parameter.strides = convolution2d.Strides(1, 1)
convolution3.parameter.nKernels = 384
convolution3.parameter.weightsInitializer = gaussian.Batch(0, 0.01)
convolution3.parameter.biasesInitializer = uniform.Batch(0, 0)
# relu
relu3 = relu.Batch()
# convolution: 3x3@384 + 2x2s
convolution4 = convolution2d.Batch()
convolution4.parameter.kernelSizes = convolution2d.KernelSizes(3, 3)
convolution4.parameter.paddings = convolution2d.Paddings(1, 1)
convolution4.parameter.strides = convolution2d.Strides(1, 1)
convolution4.parameter.nKernels = 384
convolution4.parameter.nGroups = 2
convolution4.parameter.weightsInitializer = gaussian.Batch(0, 0.01)
convolution4.parameter.biasesInitializer = uniform.Batch(0, 0)
# relu
relu4 = relu.Batch()
# convolution: 3x3@256 + 2x2s
convolution5 = convolution2d.Batch()
convolution5.parameter.kernelSizes = convolution2d.KernelSizes(3, 3)
convolution5.parameter.paddings = convolution2d.Paddings(1, 1)
convolution5.parameter.strides = convolution2d.Strides(1, 1)
convolution5.parameter.nKernels = 256
convolution5.parameter.nGroups = 2
convolution5.parameter.weightsInitializer = gaussian.Batch(0, 0.01)
convolution5.parameter.biasesInitializer = uniform.Batch(0, 0)
# relu
relu5 = relu.Batch()
# pooling: 3x3 + 2x2s
maxpooling5 = maximum_pooling2d.Batch(4)
maxpooling5.parameter.kernelSizes = pooling2d.KernelSizes(3, 3)
maxpooling5.parameter.paddings = pooling2d.Paddings(0, 0)
maxpooling5.parameter.strides = pooling2d.Strides(2, 2)
# fullyconnected: n = 4096
fullyconnected6 = fullyconnected.Batch(4096)
fullyconnected6.parameter.weightsInitializer = gaussian.Batch(0, 0.01)
fullyconnected6.parameter.biasesInitializer = uniform.Batch(0, 0)
# relu
relu6 = relu.Batch()
# dropout: p = 0.5
dropout6 = dropout.Batch()
dropout6.parameter.retainRatio = 0.5
# fullyconnected: n = 4096
fullyconnected7 = fullyconnected.Batch(4096)
fullyconnected7.parameter.weightsInitializer = gaussian.Batch(0, 0.005)
fullyconnected7.parameter.biasesInitializer = uniform.Batch(0, 0)
# relu
relu7 = relu.Batch()
# dropout: p = 0.5
dropout7 = dropout.Batch()
dropout7.parameter.retainRatio = 0.5
# fullyconnected: n = 1000
fullyconnected8 = fullyconnected.Batch(1000)
fullyconnected8.parameter.weightsInitializer = gaussian.Batch(0, 0.01)
fullyconnected8.parameter.biasesInitializer = uniform.Batch(0, 0)
# softmax + crossentropy loss
softmax = layers.loss.softmax_cross.Batch()
# Create AlexNet Topology
topology = training.Topology()
conv1 = topology.add(convolution1 )
r1 = topology.add(relu1 )
l1 = topology.add(lrn1 )
pool1 = topology.add(maxpooling1 )
conv2 = topology.add(convolution2 )
r2 = topology.add(relu2 )
l2 = topology.add(lrn2 )
pool2 = topology.add(maxpooling2 )
conv3 = topology.add(convolution3 )
r3 = topology.add(relu3 )
conv4 = topology.add(convolution4 )
r4 = topology.add(relu4 )
conv5 = topology.add(convolution5 )
r5 = topology.add(relu5 )
pool5 = topology.add(maxpooling5 )
fc6 = topology.add(fullyconnected6)
r6 = topology.add(relu6 )
drop6 = topology.add(dropout6 )
fc7 = topology.add(fullyconnected7)
r7 = topology.add(relu7 )
drop7 = topology.add(dropout7 )
fc8 = topology.add(fullyconnected8)
sm = topology.add(softmax )
topology.get(conv1).addNext(r1 )
topology.get(r1 ).addNext(l1 )
topology.get(l1 ).addNext(pool1)
topology.get(pool1).addNext(conv2)
topology.get(conv2).addNext(r2 )
topology.get(r2 ).addNext(l2 )
topology.get(l2 ).addNext(pool2)
topology.get(pool2).addNext(conv3)
topology.get(conv3).addNext(r3 )
topology.get(r3 ).addNext(conv4)
topology.get(conv4).addNext(r4 )
topology.get(r4 ).addNext(conv5)
topology.get(conv5).addNext(r5 )
topology.get(r5 ).addNext(pool5)
topology.get(pool5).addNext(fc6 )
topology.get(fc6 ).addNext(r6 )
topology.get(r6 ).addNext(drop6)
topology.get(drop6).addNext(fc7 )
topology.get(fc7 ).addNext(r7 )
topology.get(r7 ).addNext(drop7)
topology.get(drop7).addNext(fc8 )
topology.get(fc8 ).addNext(sm )
return topology
def configureNet():
# Create convolution layer
convolution1 = convolution2d.Batch()
convolution1.parameter.kernelSizes = convolution2d.KernelSizes(3, 3)
convolution1.parameter.strides = convolution2d.Strides(1, 1)
convolution1.parameter.nKernels = 32
convolution1.parameter.weightsInitializer = initializers.xavier.Batch()
convolution1.parameter.biasesInitializer = initializers.uniform.Batch(0, 0)
# Create pooling layer
maxpooling1 = maximum_pooling2d.Batch(4)
maxpooling1.parameter.kernelSizes = pooling2d.KernelSizes(2, 2)
maxpooling1.parameter.paddings = pooling2d.Paddings(0, 0)
maxpooling1.parameter.strides = pooling2d.Strides(2, 2)
# Create convolution layer
convolution2 = convolution2d.Batch()
convolution2.parameter.kernelSizes = convolution2d.KernelSizes(5, 5)
convolution2.parameter.strides = convolution2d.Strides(1, 1)
convolution2.parameter.nKernels = 64
convolution2.parameter.weightsInitializer = initializers.xavier.Batch()
convolution2.parameter.biasesInitializer = initializers.uniform.Batch(0, 0)
# Create pooling layer
maxpooling2 = maximum_pooling2d.Batch(4)
maxpooling2.parameter.kernelSizes = pooling2d.KernelSizes(2, 2)
maxpooling2.parameter.paddings = pooling2d.Paddings(0, 0)
maxpooling2.parameter.strides = pooling2d.Strides(2, 2)
# Create fullyconnected layer
fullyconnected3 = fullyconnected.Batch(256)
fullyconnected3.parameter.weightsInitializer = initializers.xavier.Batch()
fullyconnected3.parameter.biasesInitializer = initializers.uniform.Batch(
0, 0)
# Create ReLU layer
relu3 = relu.Batch()
# Create fully connected layer
fullyconnected4 = fullyconnected.Batch(10)
fullyconnected4.parameter.weightsInitializer = initializers.xavier.Batch()
fullyconnected4.parameter.biasesInitializer = initializers.uniform.Batch(
0, 0)
# Create Softmax layer
softmax = loss.softmax_cross.Batch()
# Create LeNet Topology
topology = training.Topology()
conv1 = topology.add(convolution1)
pool1 = topology.add(maxpooling1)
topology.get(conv1).addNext(pool1)
conv2 = topology.add(convolution2)
topology.get(pool1).addNext(conv2)
pool2 = topology.add(maxpooling2)
topology.get(conv2).addNext(pool2)
fc3 = topology.add(fullyconnected3)
topology.get(pool2).addNext(fc3)
r3 = topology.add(relu3)
topology.get(fc3).addNext(r3)
fc4 = topology.add(fullyconnected4)
topology.get(r3).addNext(fc4)
sm1 = topology.add(softmax)
topology.get(fc4).addNext(sm1)
return topology
def configureNet():
topology = training.Topology()
# convolution(conv1/7x7_s2): 7x7@64 + 2x2s
conv1_7x7_s2 = convolution2d.Batch()
conv1_7x7_s2.parameter.nKernels = 64
conv1_7x7_s2.parameter.kernelSizes = convolution2d.KernelSizes(7, 7)
conv1_7x7_s2.parameter.strides = convolution2d.Strides(2, 2)
conv1_7x7_s2.parameter.paddings = convolution2d.Paddings(3, 3)
conv1_7x7_s2.parameter.weightsInitializer = xavier.Batch()
conv1_7x7_s2.parameter.biasesInitializer = uniform.Batch(0.2, 0.2)
conv1_7x7_s2_id = topology.add(conv1_7x7_s2)
# relu(conv1/relu_7x7)
conv1_relu_7x7 = relu.Batch()
conv1_relu_7x7_id = topology.add(conv1_relu_7x7)
# pooling(pool1/3x3_s2): 3x3 + 2x2s
pool1_3x3_s2 = maximum_pooling2d.Batch(4)
pool1_3x3_s2.parameter.kernelSizes = pooling2d.KernelSizes(3, 3)
pool1_3x3_s2.parameter.strides = pooling2d.Strides(2, 2)
pool1_3x3_s2_id = topology.add(pool1_3x3_s2)
# lrn(pool1/norm1): alpha=0.0001, beta=0.75, local_size=5
pool1_norm1 = lrn.Batch()
pool1_norm1.parameter.kappa = 1
pool1_norm1.parameter.nAdjust = 5
pool1_norm1.parameter.beta = 0.75
pool1_norm1.parameter.alpha = 0.0001 / pool1_norm1.parameter.nAdjust
pool1_norm1_id = topology.add(pool1_norm1)
# convolution(conv2/3x3_reduce): 1x1@64 + 1x1s
conv2_3x3_reduce = convolution2d.Batch()
conv2_3x3_reduce.parameter.nKernels = 64
conv2_3x3_reduce.parameter.kernelSizes = convolution2d.KernelSizes(1, 1)
conv2_3x3_reduce.parameter.strides = convolution2d.Strides(1, 1)
conv2_3x3_reduce.parameter.paddings = convolution2d.Paddings(0, 0)
conv2_3x3_reduce.parameter.weightsInitializer = xavier.Batch()
conv2_3x3_reduce.parameter.biasesInitializer = uniform.Batch(0.2, 0.2)
conv2_3x3_reduce_id = topology.add(conv2_3x3_reduce)
# convolution(conv2/relu_3x3_reduce)
conv2_relu_3x3_reduce = relu.Batch()
conv2_relu_3x3_reduce_id = topology.add(conv2_relu_3x3_reduce)
# convolution(conv2/3x3): 3x3@192 + 1x1s
conv2_3x3 = convolution2d.Batch()
conv2_3x3.parameter.nKernels = 192
conv2_3x3.parameter.kernelSizes = convolution2d.KernelSizes(3, 3)
conv2_3x3.parameter.strides = convolution2d.Strides(1, 1)
conv2_3x3.parameter.paddings = convolution2d.Paddings(1, 1)
conv2_3x3.parameter.weightsInitializer = xavier.Batch()
conv2_3x3.parameter.biasesInitializer = uniform.Batch(0.2, 0.2)
conv2_3x3_id = topology.add(conv2_3x3)
# relu(conv2/relu_3x3)
conv2_relu_3x3 = relu.Batch()
conv2_relu_3x3_id = topology.add(conv2_relu_3x3)
# lrn(conv2/norm2): alpha=0.0001, beta=0.75, local_size=5
conv2_norm2 = lrn.Batch()
conv2_norm2.parameter.kappa = 1
conv2_norm2.parameter.nAdjust = 5
conv2_norm2.parameter.beta = 0.75
conv2_norm2.parameter.alpha = 0.0001 / conv2_norm2.parameter.nAdjust
conv2_norm2_id = topology.add(conv2_norm2)
# pooling(pool2/3x3_s2): 3x3 + 1x1s
pool2_3x3_s2 = maximum_pooling2d.Batch(4)
pool2_3x3_s2.parameter.kernelSizes = pooling2d.KernelSizes(3, 3)
pool2_3x3_s2.parameter.strides = pooling2d.Strides(2, 2)
pool2_3x3_s2_id = topology.add(pool2_3x3_s2)
# inception module, convolution filters = (64, 96, 128, 16, 32, 32)
inception1 = configureInception(64, 96, 128, 16, 32, 32)
inception1_end_id, inception1_start_id = topology.add(inception1)
# inception module, convolution filters = (128, 128, 192, 32, 96, 64)
inception2 = configureInception(128, 128, 192, 32, 96, 64)
inception2_end_id, inception2_start_id = topology.add(inception2)
# pooling(pool3/3x3_s2): 3x3 + 2x2s
pool3_3x3_s2 = maximum_pooling2d.Batch(4)
pool3_3x3_s2.parameter.kernelSizes = pooling2d.KernelSizes(3, 3)
pool3_3x3_s2.parameter.strides = pooling2d.Strides(2, 2)
pool3_3x3_s2_id = topology.add(pool3_3x3_s2)
# inception module, convolution filters = (192, 96, 208, 16, 48, 64)
inception3 = configureInception(192, 96, 208, 16, 48, 64)
inception3_end_id, inception3_start_id = topology.add(inception3)
# inception module, convolution filters = (160, 112, 224, 24, 64, 64)
inception4 = configureInception(160, 112, 224, 24, 64, 64,
True) # has loss branch
inception4_end_id, inception4_start_id = topology.add(inception4)
# loss branch 1
loss1Branch = configureLossLayers()
loss1_end_id, loss1_start_id = topology.add(loss1Branch)
# inception module, convolution filters = (128, 128, 256, 24, 64, 64)
inception5 = configureInception(128, 128, 256, 24, 64, 64)
inception5_end_id, inception5_start_id = topology.add(inception5)
# inception module, convolution filters = (112, 144, 288, 32, 64, 64)
inception6 = configureInception(112, 144, 288, 32, 64, 64)
inception6_end_id, inception6_start_id = topology.add(inception6)
# inception module, convolution filters = (256, 160, 320, 32, 128, 128)
inception7 = configureInception(256, 160, 320, 32, 128, 128,
True) # has loss branch
inception7_end_id, inception7_start_id = topology.add(inception7)
# loss branch 2
loss2Branch = configureLossLayers()
loss2_end_id, loss2_start_id = topology.add(loss2Branch)
# pooling(pool4_3x3_s2): 3x3 + 2x2s
pool4_3x3_s2 = maximum_pooling2d.Batch(4)
pool4_3x3_s2.parameter.kernelSizes = pooling2d.KernelSizes(3, 3)
pool4_3x3_s2.parameter.strides = pooling2d.Strides(2, 2)
pool4_3x3_s2_id = topology.add(pool4_3x3_s2)
# inception module, convolution filters = (256, 160, 320, 32, 128, 128)
inception8 = configureInception(256, 160, 320, 32, 128, 128)
inception8_end_id, inception8_start_id = topology.add(inception8)
# inception module, convolution filters = (384, 192, 384, 48, 128, 128)
inception9 = configureInception(384, 192, 384, 48, 128, 128)
inception9_end_id, inception9_start_id = topology.add(inception9)
# pooling(pool5/7x7_s1): 7x7 + 1x1s
pool5_7x7_s1 = average_pooling2d.Batch(4)
pool5_7x7_s1.parameter.kernelSizes = pooling2d.KernelSizes(7, 7)
pool5_7x7_s1.parameter.strides = pooling2d.Strides(1, 1)
pool5_7x7_s1_id = topology.add(pool5_7x7_s1)
# dropout(pool5/drop_7x7_s1): p = 0.4
pool5_drop_7x7_s1 = dropout.Batch()
pool5_drop_7x7_s1.parameter.retainRatio = 0.4
pool5_drop_7x7_s1_id = topology.add(pool5_drop_7x7_s1)
# fullyconnected(loss3/classifier): n = 1000
loss3_classifier = fullyconnected.Batch(1000)
loss3_classifier.parameter.weightsInitializer = xavier.Batch()
loss3_classifier.parameter.biasesInitializer = uniform.Batch(0.0, 0.0)
loss3_classifier_id = topology.add(loss3_classifier)
# softmax + crossentropy loss (loss3/loss3)
loss3_loss3 = loss.softmax_cross.Batch()
loss3_end_id = topology.add(loss3_loss3)
topology.get(conv1_7x7_s2_id).addNext(conv1_relu_7x7_id)
topology.get(conv1_relu_7x7_id).addNext(pool1_3x3_s2_id)
topology.get(pool1_3x3_s2_id).addNext(pool1_norm1_id)
topology.get(pool1_norm1_id).addNext(conv2_3x3_reduce_id)
topology.get(conv2_3x3_reduce_id).addNext(conv2_relu_3x3_reduce_id)
topology.get(conv2_relu_3x3_reduce_id).addNext(conv2_3x3_id)
topology.get(conv2_3x3_id).addNext(conv2_relu_3x3_id)
topology.get(conv2_relu_3x3_id).addNext(conv2_norm2_id)
topology.get(conv2_norm2_id).addNext(pool2_3x3_s2_id)
topology.get(pool2_3x3_s2_id).addNext(inception1_start_id)
topology.get(inception1_end_id).addNext(inception2_start_id)
topology.get(inception2_end_id).addNext(pool3_3x3_s2_id)
topology.get(pool3_3x3_s2_id).addNext(inception3_start_id)
topology.get(inception3_end_id).addNext(inception4_start_id)
topology.get(inception4_start_id).addNext(loss1_start_id)
topology.get(inception4_end_id).addNext(inception5_start_id)
topology.get(inception5_end_id).addNext(inception6_start_id)
topology.get(inception6_end_id).addNext(inception7_start_id)
topology.get(inception7_start_id).addNext(loss2_start_id)
topology.get(inception7_end_id).addNext(pool4_3x3_s2_id)
topology.get(pool4_3x3_s2_id).addNext(inception8_start_id)
topology.get(inception8_end_id).addNext(inception9_start_id)
topology.get(inception9_end_id).addNext(pool5_7x7_s1_id)
topology.get(pool5_7x7_s1_id).addNext(pool5_drop_7x7_s1_id)
topology.get(pool5_drop_7x7_s1_id).addNext(loss3_classifier_id)
topology.get(loss3_classifier_id).addNext(loss3_end_id)
lastLayers = [loss1_end_id, loss2_end_id, loss3_end_id]
return topology, lastLayers