def __init__(self, **kwargs):
super(OneLayerBrain, self).__init__(**kwargs)
self.attach(
ConvolutionLayer(ksize=[5, 5],
strides=[1, 1],
padding="SAME",
in_channel_num=1,
out_channel_num=32,
name="conv1")
)
self.attach(ReLULayer(name="relu1"))
self.attach(
PoolingLayer(ksize=[5, 5],
strides=[5, 5],
padding="SAME",
name="pool1")
)
# self.attach(InnerProductLayer(in_channel_num=1152, out_channel_num=10))
# Since torch returns None when differentiating constant, we disable
# the last layer's bias to avoid creating None.
self.attach(InnerProductLayer(in_channel_num=1152, out_channel_num=1, initial_bias_value=None))
self.attach(ReshapeLayer(name="ip"))
self.attach(HingeLossLayer(
inputs=[
{"name": "ip", "idxs": [0]},
{"name": "system_in", "idxs": [1]}],
))
self.attach(BinaryAccuracy(hinge_loss_label=True, inputs=[{"name": "ip"}, {"name": "system_in", "idxs": [1]}]))
def __init__(self, **kwargs):
super(LeNet, self).__init__(**kwargs)
self.attach(ConvolutionLayer(ksize=[5, 5],
strides=[1, 1],
padding="SAME",
in_channel_num=1,
out_channel_num=32,
name="conv1"))
self.attach(ReLULayer(name="relu1"))
self.attach(PoolingLayer(ksize=[2, 2],
strides=[2, 2],
padding="SAME",
name="pool1"))
self.attach(ConvolutionLayer(ksize=[5, 5],
strides=[1, 1],
padding="SAME",
in_channel_num=32,
out_channel_num=64,
name="conv2"))
self.attach(ReLULayer(name="relu2"))
self.attach(PoolingLayer(ksize=[5, 5],
strides=[2, 2],
padding="SAME",
name="pool2"))
self.attach(InnerProductLayer(
in_channel_num=2304,
out_channel_num=512,
name="ip1"))
self.attach(ReLULayer(name="relu3"))
self.attach(InnerProductLayer(
in_channel_num=512,
out_channel_num=1,
initial_bias_value=None,
name="ip2"))
self.attach(ReshapeLayer(name="ip"))
self.attach(HingeLossLayer(
inputs=[
{"name": "ip", "idxs": [0]},
{"name": "system_in", "idxs": [1]}],
))
self.attach(BinaryAccuracy(hinge_loss_label=True, inputs=[{"name": "ip"}, {"name": "system_in", "idxs": [1]}]))
def __init__(self, *args, **kwargs):
super(VGG11, self).__init__(*args, **kwargs)
channel_num = [32, 32, 64, 64, 128, 128, 256, 256, 512, 512]
for i in range(10):
self.attach(ConvolutionLayer(3, 1, "SAME",
in_channel_num=1 if i == 0 else channel_num[i-1],
initial_bias_value=None,
out_channel_num=channel_num[i]))
# Uncomment to add BN.
# self.attach(BN(channel_num[i])),
self.attach(ReLULayer())
if i != 0 and i % 2 == 0:
self.attach(MaxPoolingLayer(ksize=2, strides=2, padding="VALID"))
self.attach(ReshapeLayer())
self.attach(InnerProductLayer(in_channel_num=512, out_channel_num=1, initial_bias_value=None))
self.attach(ReshapeLayer(name="ip"))
self.attach(HingeLossLayer(inputs=[{"name": "ip"}, {"name": "system_in", "idxs": [1]}]))
self.attach(BinaryAccuracy(hinge_loss_label=True, inputs=[{"name": "ip"}, {"name": "system_in", "idxs": [1]}]))
def setup():
# Set up brain
# #########################################################################
brain = Brain(name='maxout-zca-cifar10')
brain.attach(DropoutLayer(keep_prob=0.8, name='dropout1'))
brain.attach(
ConvolutionLayer([8, 8], [1, 1, 1, 1],
'SAME',
init_para={
"name": "uniform",
"range": 0.005
},
max_norm=0.9,
out_channel_num=192,
name='conv1'))
brain.attach(PoolingLayer([1, 4, 4, 1], [1, 2, 2, 1], 'SAME',
name='pool1'))
brain.attach(MaxoutLayer(name='maxout1'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout2'))
brain.attach(
ConvolutionLayer([8, 8], [1, 1, 1, 1],
'SAME',
init_para={
"name": "uniform",
"range": 0.005
},
max_norm=1.9365,
out_channel_num=384,
name='conv2'))
brain.attach(PoolingLayer([1, 4, 4, 1], [1, 2, 2, 1], 'SAME',
name='pool2'))
brain.attach(MaxoutLayer(name='maxout2'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
ConvolutionLayer([5, 5], [1, 1, 1, 1],
'SAME',
init_para={
"name": "uniform",
"range": 0.005
},
max_norm=1.9365,
out_channel_num=384,
name='conv3'))
brain.attach(PoolingLayer([1, 2, 2, 1], [1, 2, 2, 1], 'SAME',
name='pool3'))
brain.attach(MaxoutLayer(name='maxout3'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
InnerProductLayer(init_para={
"name": "uniform",
"range": 0.005
},
max_norm=1.9,
out_channel_num=2500,
name='ip1'))
brain.attach(MaxoutLayer(group_size=5, name='maxout4'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
InnerProductLayer(init_para={
"name": "uniform",
"range": 0.005
},
max_norm=1.9365,
out_channel_num=10,
name='softmax_linear'))
brain.attach(SoftmaxWithLossLayer(class_num=10, name='loss'))
# Set up a sensor.
# #########################################################################
cifar_source = Cifar10FeedSource(
name="CIFAR10",
url='http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz',
work_dir=AKID_DATA_PATH + '/cifar10',
use_zca=True,
num_train=50000,
num_val=10000)
sensor = FeedSensor(source_in=cifar_source, batch_size=128, name='data')
# Summon a survivor.
# #########################################################################
survivor = kids.Kid(sensor,
brain,
kongfus.MomentumKongFu(base_lr=0.025,
momentum=0.5,
decay_rate=0.1,
decay_epoch_num=50),
max_steps=200000)
survivor.setup()
return survivor
def setup(graph, lr):
# Set up brain
# #########################################################################
brain = GraphBrain(moving_average_decay=0.99, name='maxout-relu-cifar10')
brain.attach(
ConvolutionLayer([8, 8], [1, 1, 1, 1],
'SAME',
init_para={
"name": "truncated_normal",
"stddev": 0.005
},
wd={
"type": "l2",
"scale": 0.0005
},
out_channel_num=192,
name='conv1'))
brain.attach(PoolingLayer([1, 4, 4, 1], [1, 2, 2, 1], 'SAME',
name='pool1'))
brain.attach(CollapseOutLayer(name='maxout1'))
brain.attach(DropoutLayer(keep_prob=0.8, name='dropout1'))
brain.attach(
ConvolutionLayer([8, 8], [1, 1, 1, 1],
'SAME',
init_para={
"name": "truncated_normal",
"stddev": 0.005
},
wd={
"type": "l2",
"scale": 0.0005
},
out_channel_num=384,
name='conv2'))
brain.attach(PoolingLayer([1, 4, 4, 1], [1, 2, 2, 1], 'SAME',
name='pool2'))
brain.attach(CollapseOutLayer(name='maxout2'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout2'))
brain.attach(
ConvolutionLayer([5, 5], [1, 1, 1, 1],
'SAME',
init_para={
"name": "truncated_normal",
"stddev": 0.005
},
wd={
"type": "l2",
"scale": 0.0005
},
out_channel_num=384,
name='conv3'))
brain.attach(PoolingLayer([1, 2, 2, 1], [1, 2, 2, 1], 'SAME',
name='pool3'))
brain.attach(CollapseOutLayer(name='maxout3'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
InnerProductLayer(init_para={
"name": "truncated_normal",
"stddev": 0.005
},
wd={
"type": "l2",
"scale": 0.004
},
out_channel_num=2500,
name='ip1'))
brain.attach(CollapseOutLayer(group_size=5, name='maxout4'))
brain.attach(DropoutLayer(keep_prob=0.3, name='dropout3'))
brain.attach(
InnerProductLayer(init_para={
"name": "truncated_normal",
"stddev": 1 / 500.0
},
wd={
"type": "l2",
"scale": 0.004
},
out_channel_num=10,
name='softmax_linear'))
brain.attach(SoftmaxWithLossLayer(class_num=10, name='loss'))
# Set up a sensor.
# #########################################################################
cifar_source = Cifar10TFSource(
name="CIFAR10",
url='http://www.cs.toronto.edu/~kriz/cifar-10-binary.tar.gz',
work_dir=AKID_DATA_PATH + '/cifar10',
num_train=50000,
num_val=10000)
sensor = IntegratedSensor(source_in=cifar_source,
batch_size=128,
name='data')
sensor.attach(WhitenJoker(name="per_image_whitening"), to_val=True)
sensor.attach(WhitenJoker(name="per_image_whitening"))
# Summon a kid.
# #########################################################################
survivor = kids.Kid(sensor,
brain,
kongfus.MomentumKongFu(base_lr=lr,
momentum=0.5,
decay_rate=0.1,
decay_epoch_num=50),
log_dir="log_{}".format(lr),
max_steps=510000,
graph=graph)
survivor.setup()
return survivor
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout2'))
brain.attach(
ConvolutionLayer([5, 5], [1, 1, 1, 1],
'SAME',
stddev=0.005,
weight_decay=0.0005,
out_channel_num=384,
name='conv3'))
brain.attach(PoolingLayer([1, 2, 2, 1], [1, 2, 2, 1], 'SAME', name='pool3'))
brain.attach(MaxoutLayer(name='maxout3'))
brain.attach(DropoutLayer(keep_prob=0.5, name='dropout3'))
brain.attach(
InnerProductLayer(stddev=0.005,
weight_decay=0.004,
out_channel_num=500,
name='ip1'))
brain.attach(MaxoutLayer(group_size=5, name='maxout4'))
brain.attach(DropoutLayer(keep_prob=0.3, name='dropout3'))
brain.attach(
InnerProductLayer(stddev=1 / 500.0,
weight_decay=0,
out_channel_num=10,
name='softmax_linear'))
brain.attach(SoftmaxWithLossLayer(class_num=10, name='loss'))
# Set up a sensor.
# #########################################################################
cifar_source = Cifar10TFSource(
def cnn_block(ksize=None,
conv_padding="SAME",
initial_bias_value=0.,
init_para={
"name": "truncated_normal",
"stddev": 0.1
},
wd={
"type": "l2",
"scale": 5e-4
},
max_norm=None,
in_channel_num=32,
out_channel_num=32,
pool_size=[2, 2],
pool_stride=[2, 2],
activation={"type": "relu"},
activation_before_pooling=False,
keep_prob=None,
bn=None):
global counter
counter += 1
block = []
if ksize:
block.append(
ConvolutionLayer(ksize=ksize,
strides=[1, 1],
padding=conv_padding,
initial_bias_value=initial_bias_value,
init_para=init_para,
wd=wd,
max_norm=max_norm,
in_channel_num=in_channel_num,
out_channel_num=out_channel_num,
name="conv{}".format(counter)))
if pool_size:
block.append(
PoolingLayer(ksize=pool_size,
strides=pool_stride,
padding="SAME",
name="pool{}".format(counter)))
else:
block.append(
InnerProductLayer(out_channel_num=out_channel_num,
in_channel_num=in_channel_num,
initial_bias_value=initial_bias_value,
init_para=init_para,
wd=wd,
name="ip{}".format(counter)))
if bn:
bn_layer = BatchNormalizationLayer(name="bn{}".format(counter), **bn)
if activation:
try:
activation_type = activation["type"]
if activation_type == "relu":
layer = ReLULayer(name="relu{}".format(counter))
elif activation_type == "maxout":
layer = CollapseOutLayer(name="maxout{}".format(counter),
group_size=activation["group_size"])
elif activation_type == "ngsmax":
layer = SoftmaxNormalizationLayer(
name="ngsoftmax{}".format(counter),
group_size=activation["group_size"])
elif activation_type == "gsmax":
layer = GroupSoftmaxLayer(name="gsmax{}".format(counter),
group_size=activation["group_size"])
else:
print(("{} activation type has not been supported"
" yet.".format(activation_type)))
sys.exit(0)
except KeyError as e:
print(("{} not found. You perhaps have a typo or miss a"
" parameter.".format(e)))
sys.exit(0)
else:
layer = None
if activation_before_pooling:
for i, b in enumerate(block):
if type(b) is PoolingLayer:
if layer:
block.insert(i, layer)
if bn:
block.insert(i, bn_layer)
break
else:
# Even if there is no activation layer, which happens at the last
# readout layer, BN may still be needed.
if bn:
block.append(bn_layer)
if layer:
block.append(layer)
if keep_prob:
block.append(
DropoutLayer(keep_prob=keep_prob,
name="dropout{}".format(counter)))
return block