def setup():
# Set up brain
# #########################################################################
brain = Brain(name='maxout-zca-cifar10')
brain.attach(DropoutLayer(keep_prob=0.8, name='dropout1'))
brain.attach(
ConvolutionLayer([8, 8], [1, 1, 1, 1],
'SAME',
init_para={
"name": "uniform",
"range": 0.005
},
max_norm=0.9,
out_channel_num=192,
name='conv1'))
brain.attach(PoolingLayer([1, 4, 4, 1], [1, 2, 2, 1], 'SAME',
name='pool1'))
brain.attach(MaxoutLayer(name='maxout1'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout2'))
brain.attach(
ConvolutionLayer([8, 8], [1, 1, 1, 1],
'SAME',
init_para={
"name": "uniform",
"range": 0.005
},
max_norm=1.9365,
out_channel_num=384,
name='conv2'))
brain.attach(PoolingLayer([1, 4, 4, 1], [1, 2, 2, 1], 'SAME',
name='pool2'))
brain.attach(MaxoutLayer(name='maxout2'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
ConvolutionLayer([5, 5], [1, 1, 1, 1],
'SAME',
init_para={
"name": "uniform",
"range": 0.005
},
max_norm=1.9365,
out_channel_num=384,
name='conv3'))
brain.attach(PoolingLayer([1, 2, 2, 1], [1, 2, 2, 1], 'SAME',
name='pool3'))
brain.attach(MaxoutLayer(name='maxout3'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
InnerProductLayer(init_para={
"name": "uniform",
"range": 0.005
},
max_norm=1.9,
out_channel_num=2500,
name='ip1'))
brain.attach(MaxoutLayer(group_size=5, name='maxout4'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
InnerProductLayer(init_para={
"name": "uniform",
"range": 0.005
},
max_norm=1.9365,
out_channel_num=10,
name='softmax_linear'))
brain.attach(SoftmaxWithLossLayer(class_num=10, name='loss'))
# Set up a sensor.
# #########################################################################
cifar_source = Cifar10FeedSource(
name="CIFAR10",
url='http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz',
work_dir=AKID_DATA_PATH + '/cifar10',
use_zca=True,
num_train=50000,
num_val=10000)
sensor = FeedSensor(source_in=cifar_source, batch_size=128, name='data')
# Summon a survivor.
# #########################################################################
survivor = kids.Kid(sensor,
brain,
kongfus.MomentumKongFu(base_lr=0.025,
momentum=0.5,
decay_rate=0.1,
decay_epoch_num=50),
max_steps=200000)
survivor.setup()
return survivor
def setup(graph, lr):
# Set up brain
# #########################################################################
brain = GraphBrain(moving_average_decay=0.99, name='maxout-relu-cifar10')
brain.attach(
ConvolutionLayer([8, 8], [1, 1, 1, 1],
'SAME',
init_para={
"name": "truncated_normal",
"stddev": 0.005
},
wd={
"type": "l2",
"scale": 0.0005
},
out_channel_num=192,
name='conv1'))
brain.attach(PoolingLayer([1, 4, 4, 1], [1, 2, 2, 1], 'SAME',
name='pool1'))
brain.attach(CollapseOutLayer(name='maxout1'))
brain.attach(DropoutLayer(keep_prob=0.8, name='dropout1'))
brain.attach(
ConvolutionLayer([8, 8], [1, 1, 1, 1],
'SAME',
init_para={
"name": "truncated_normal",
"stddev": 0.005
},
wd={
"type": "l2",
"scale": 0.0005
},
out_channel_num=384,
name='conv2'))
brain.attach(PoolingLayer([1, 4, 4, 1], [1, 2, 2, 1], 'SAME',
name='pool2'))
brain.attach(CollapseOutLayer(name='maxout2'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout2'))
brain.attach(
ConvolutionLayer([5, 5], [1, 1, 1, 1],
'SAME',
init_para={
"name": "truncated_normal",
"stddev": 0.005
},
wd={
"type": "l2",
"scale": 0.0005
},
out_channel_num=384,
name='conv3'))
brain.attach(PoolingLayer([1, 2, 2, 1], [1, 2, 2, 1], 'SAME',
name='pool3'))
brain.attach(CollapseOutLayer(name='maxout3'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
InnerProductLayer(init_para={
"name": "truncated_normal",
"stddev": 0.005
},
wd={
"type": "l2",
"scale": 0.004
},
out_channel_num=2500,
name='ip1'))
brain.attach(CollapseOutLayer(group_size=5, name='maxout4'))
brain.attach(DropoutLayer(keep_prob=0.3, name='dropout3'))
brain.attach(
InnerProductLayer(init_para={
"name": "truncated_normal",
"stddev": 1 / 500.0
},
wd={
"type": "l2",
"scale": 0.004
},
out_channel_num=10,
name='softmax_linear'))
brain.attach(SoftmaxWithLossLayer(class_num=10, name='loss'))
# Set up a sensor.
# #########################################################################
cifar_source = Cifar10TFSource(
name="CIFAR10",
url='http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz',
work_dir=AKID_DATA_PATH + '/cifar10',
num_train=50000,
num_val=10000)
sensor = IntegratedSensor(source_in=cifar_source,
batch_size=128,
name='data')
sensor.attach(WhitenJoker(name="per_image_whitening"), to_val=True)
sensor.attach(WhitenJoker(name="per_image_whitening"))
# Summon a kid.
# #########################################################################
survivor = kids.Kid(sensor,
brain,
kongfus.MomentumKongFu(base_lr=lr,
momentum=0.5,
decay_rate=0.1,
decay_epoch_num=50),
log_dir="log_{}".format(lr),
max_steps=510000,
graph=graph)
survivor.setup()
return survivor
# because of jpeg decoding, the shape is determined dynamically, thus
# we manually set shape here.
data = tf.reshape(data, shape=[224, 224, 3])
self._data = data
sensor.attach(AdhocJoker(name="to_1_-1"))
sensor.attach(AdhocJoker(name="to_1_-1"), to_val=True)
def update_lr(kid):
if kid.epoch < 30:
kid.kongfu.lr_value = 0.1
elif kid.epoch < 60:
kid.kongfu.lr_value = 0.01
else:
kid.kongfu.lr_value = 0.001
# Summon a survivor.
# #########################################################################
from akid import LearningRateScheme
kid = kids.Kid(
sensor,
brain,
kongfus.MomentumKongFu(lr_scheme={"name": LearningRateScheme.placeholder}),
engine={"name": "data_parallel", "num_gpu": 8},
# engine={"name": "single"},
log_dir="log",
max_epoch=100)
kid.hooks.on_batch_begin.append(update_lr)
kid.setup()
kid.practice()
name='softmax_linear'))
brain.attach(SoftmaxWithLossLayer(class_num=10, name='loss'))
# Set up a sensor.
# #########################################################################
cifar_source = Cifar10TFSource(
name="CIFAR10",
url='http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz',
work_dir=AKID_DATA_PATH + '/cifar10',
num_train=50000,
num_val=10000)
sensor = IntegratedSensor(source_in=cifar_source, batch_size=128, name='data')
sensor.attach(WhitenJoker(name="per_image_whitening"), to_val=True)
sensor.attach(WhitenJoker(name="per_image_whitening"))
# Summon a survivor.
# #########################################################################
survivor = kids.Kid(sensor,
brain,
kongfus.MomentumKongFu(base_lr=0.01,
momentum=0.5,
decay_rate=0.1,
decay_epoch_num=50),
max_steps=510000)
# Start training
# #######################################################################
survivor.practice(continue_from_chk_point=False)