def resnet(units, num_stage, filter_list, num_class, data_type, bottle_neck=True, bn_mom=0.9, workspace=512, memonger=False):
"""Return ResNet symbol of cifar10 and imagenet
Parameters
----------
units : list
Number of units in each stage
num_stage : int
Number of stage
filter_list : list
Channel size of each stage
num_class : int
Ouput size of symbol
dataset : str
Dataset type, only cifar10 and imagenet supports
workspace : int
Workspace used in convolution operator
"""
num_unit = len(units)
assert(num_unit == num_stage)
# data = mxc.Variable(name='data')
data = 'data'
data = mxc.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data')
if data_type == 'cifar10':
body = mxc.Convolution(data=data, num_filter=filter_list[0], kernel=(3, 3), stride=(1,1), pad=(1, 1),
no_bias=True, name="conv0", workspace=workspace)
elif data_type == 'imagenet':
body = mxc.Convolution(data=data, num_filter=filter_list[0], kernel=(7, 7), stride=(2,2), pad=(3, 3),
no_bias=True, name="conv0", workspace=workspace)
body = mxc.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0')
body = mxc.Activation(data=body, act_type='relu', name='relu0')
body = mxc.Pooling(data=body, name="pool0", kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max')
else:
raise ValueError("do not support {} yet".format(data_type))
for i in range(num_stage):
body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False,
name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, workspace=workspace,
memonger=memonger)
for j in range(units[i]-1):
body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i + 1, j + 2),
bottle_neck=bottle_neck, workspace=workspace, memonger=memonger)
bn1 = mxc.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1')
relu1 = mxc.Activation(data=bn1, act_type='relu', name='relu1')
# Although kernel is not used here when global_pool=True, we should put one
# pool1 = mxc.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1')
pool1 = mxc.Pooling(data=relu1, kernel=(7, 7), pool_type='avg', name='pool1')
flat = mxc.Flatten(data=pool1, name='flatten')
fc1 = mxc.FullyConnected(data=flat, num_hidden=num_class, name='fc1')
def get_conv(name, data, num_filter, kernel, stride, pad, with_relu,
bn_momentum):
conv = mxc.Convolution(name=name,
data=data,
num_filter=num_filter,
kernel=kernel,
stride=stride,
pad=pad,
no_bias=True)
bn = mxc.BatchNorm(
name=name + '_bn',
data=conv,
fix_gamma=False,
momentum=bn_momentum,
# Same with https://github.com/soumith/cudnn.torch/blob/master/BatchNormalization.lua
# cuDNN v5 don't allow a small eps of 1e-5
eps=2e-5)
return (
# It's better to remove ReLU here
# https://github.com/gcr/torch-residual-networks
mxc.Activation(name=name +
'_relu', data=bn, act_type='relu') if with_relu else bn)
def ConvFactory(data, num_filter, kernel, stride=(1,1), pad=(0, 0), name=None, suffix=''):
conv = mxc.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, name='conv_%s%s' %(name, suffix))
bn = mxc.BatchNorm(data=conv, name='bn_%s%s' %(name, suffix))
act = mxc.Activation(data=bn, act_type='relu', name='relu_%s%s' %(name, suffix))
return act
def make_block(name, data, num_filter, dim_match, bn_momentum):
conv1 = get_conv(name=name + '_conv1',
data=data,
num_filter=num_filter,
kernel=(3, 3),
stride=(1, 1) if dim_match else (2, 2),
pad=(1, 1),
with_relu=True,
bn_momentum=bn_momentum)
conv2 = get_conv(name=name + '_conv2',
data=conv1,
num_filter=num_filter,
kernel=(3, 3),
stride=(1, 1),
pad=(1, 1),
with_relu=False,
bn_momentum=bn_momentum)
if dim_match:
shortcut = data
else:
# Like http://ethereon.github.io/netscope/#/gist/db945b393d40bfa26006
# Test accuracy 0.922+ on CIFAR10 with 56 layers
# shortcut = get_conv(
# name=name + '_proj',
# data=data,
# num_filter=num_filter,
# kernel=(1, 1),
# stride=(2, 2),
# pad=(0, 0),
# with_relu=False,
# bn_momentum=bn_momentum
# )
# Type A shortcut
# Note we use kernel (2, 2) rather than (1, 1) and a custom initializer
# in train_cifar10_resnet.py
# Test accuracy 0.918 on CIFAR10 with 56 layers and kernel (1, 1)
# TODO(Answeror): Don't know why (1, 1) got much lower accuracy
shortcut = mxc.Convolution(name=name + '_proj',
data=data,
num_filter=num_filter,
kernel=(2, 2),
stride=(2, 2),
pad=(0, 0),
no_bias=True)
# Same with above, but ugly
# Mxnet don't have nn.Padding as that in
# https://github.com/gcr/torch-residual-networks/blob/master/residual-layers.lua
# shortcut = mxc.Pooling(
# data=data,
# name=name + '_pool',
# kernel=(2, 2),
# stride=(2, 2),
# pool_type='avg'
# )
# shortcut = mxc.Concat(
# shortcut,
# mxc.minimum(shortcut + 1, 0),
# num_args=2
# )
fused = mxc.Add(name=name + '_fused', bottoms=[shortcut, conv2])
return mxc.Activation(name=name + '_relu', data=fused, act_type='relu')
def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, bn_mom=0.9, workspace=512, memonger=False):
"""Return ResNet Unit symbol for building ResNet
Parameters
----------
data : str
Input data
num_filter : int
Number of output channels
bnf : int
Bottle neck channels factor with regard to num_filter
stride : tupe
Stride used in convolution
dim_match : Boolen
True means channel number between input and output is the same, otherwise means differ
name : str
Base name of the operators
workspace : int
Workspace used in convolution operator
"""
if bottle_neck:
# the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper
bn1 = mxc.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1')
act1 = mxc.Activation(data=bn1, act_type='relu', name=name + '_relu1')
conv1 = mxc.Convolution(data=act1, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0),
no_bias=True, workspace=workspace, name=name + '_conv1')
bn2 = mxc.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2')
act2 = mxc.Activation(data=bn2, act_type='relu', name=name + '_relu2')
conv2 = mxc.Convolution(data=act2, num_filter=int(num_filter*0.25), kernel=(3,3), stride=stride, pad=(1,1),
no_bias=True, workspace=workspace, name=name + '_conv2')
bn3 = mxc.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3')
act3 = mxc.Activation(data=bn3, act_type='relu', name=name + '_relu3')
conv3 = mxc.Convolution(data=act3, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True,
workspace=workspace, name=name + '_conv3')
if dim_match:
shortcut = data
else:
shortcut = mxc.Convolution(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True,
workspace=workspace, name=name+'_sc')
if memonger:
shortcut._set_attr(mirror_stage='True')
fused = mxc.Add(
name = name + '_fused',
bottoms = [conv3, shortcut])
return fused
else:
bn1 = mxc.BatchNorm(data=data, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1')
act1 = mxc.Activation(data=bn1, act_type='relu', name=name + '_relu1')
conv1 = mxc.Convolution(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1),
no_bias=True, workspace=workspace, name=name + '_conv1')
bn2 = mxc.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2')
act2 = mxc.Activation(data=bn2, act_type='relu', name=name + '_relu2')
conv2 = mxc.Convolution(data=act2, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1),
no_bias=True, workspace=workspace, name=name + '_conv2')
if dim_match:
shortcut = data
else:
shortcut = mxc.Convolution(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True,
workspace=workspace, name=name+'_sc')
if memonger:
shortcut._set_attr(mirror_stage='True')
fused = mxc.Add(
name = name + '_fused',
bottoms = [conv2, shortcut])
return fused
def Conv(data, num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name=None, suffix=''):
conv = mxc.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, name='%s%s_conv2d' %(name, suffix))
bn = mxc.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix))
act = mxc.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix))
return act