def resnet(units, num_stage, filter_list, num_class, data_type, bottle_neck=True, bn_mom=0.9, workspace=512, memonger=False): """Return ResNet symbol of cifar10 and imagenet Parameters ---------- units : list Number of units in each stage num_stage : int Number of stage filter_list : list Channel size of each stage num_class : int Ouput size of symbol dataset : str Dataset type, only cifar10 and imagenet supports workspace : int Workspace used in convolution operator """ num_unit = len(units) assert(num_unit == num_stage) # data = mxc.Variable(name='data') data = 'data' data = mxc.BatchNorm(data=data, fix_gamma=True, eps=2e-5, momentum=bn_mom, name='bn_data') if data_type == 'cifar10': body = mxc.Convolution(data=data, num_filter=filter_list[0], kernel=(3, 3), stride=(1,1), pad=(1, 1), no_bias=True, name="conv0", workspace=workspace) elif data_type == 'imagenet': body = mxc.Convolution(data=data, num_filter=filter_list[0], kernel=(7, 7), stride=(2,2), pad=(3, 3), no_bias=True, name="conv0", workspace=workspace) body = mxc.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn0') body = mxc.Activation(data=body, act_type='relu', name='relu0') body = mxc.Pooling(data=body, name="pool0", kernel=(3, 3), stride=(2,2), pad=(1,1), pool_type='max') else: raise ValueError("do not support {} yet".format(data_type)) for i in range(num_stage): body = residual_unit(body, filter_list[i+1], (1 if i==0 else 2, 1 if i==0 else 2), False, name='stage%d_unit%d' % (i + 1, 1), bottle_neck=bottle_neck, workspace=workspace, memonger=memonger) for j in range(units[i]-1): body = residual_unit(body, filter_list[i+1], (1,1), True, name='stage%d_unit%d' % (i + 1, j + 2), bottle_neck=bottle_neck, workspace=workspace, memonger=memonger) bn1 = mxc.BatchNorm(data=body, fix_gamma=False, eps=2e-5, momentum=bn_mom, name='bn1') relu1 = mxc.Activation(data=bn1, act_type='relu', name='relu1') # Although kernel is not used here when global_pool=True, we should put one # pool1 = mxc.Pooling(data=relu1, global_pool=True, kernel=(7, 7), pool_type='avg', name='pool1') pool1 = mxc.Pooling(data=relu1, kernel=(7, 7), pool_type='avg', name='pool1') flat = mxc.Flatten(data=pool1, name='flatten') fc1 = mxc.FullyConnected(data=flat, num_hidden=num_class, name='fc1')
def get_conv(name, data, num_filter, kernel, stride, pad, with_relu, bn_momentum): conv = mxc.Convolution(name=name, data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, no_bias=True) bn = mxc.BatchNorm( name=name + '_bn', data=conv, fix_gamma=False, momentum=bn_momentum, # Same with https://github.com/soumith/cudnn.torch/blob/master/BatchNormalization.lua # cuDNN v5 don't allow a small eps of 1e-5 eps=2e-5) return ( # It's better to remove ReLU here # https://github.com/gcr/torch-residual-networks mxc.Activation(name=name + '_relu', data=bn, act_type='relu') if with_relu else bn)
def ConvFactory(data, num_filter, kernel, stride=(1,1), pad=(0, 0), name=None, suffix=''): conv = mxc.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, name='conv_%s%s' %(name, suffix)) bn = mxc.BatchNorm(data=conv, name='bn_%s%s' %(name, suffix)) act = mxc.Activation(data=bn, act_type='relu', name='relu_%s%s' %(name, suffix)) return act
def make_block(name, data, num_filter, dim_match, bn_momentum): conv1 = get_conv(name=name + '_conv1', data=data, num_filter=num_filter, kernel=(3, 3), stride=(1, 1) if dim_match else (2, 2), pad=(1, 1), with_relu=True, bn_momentum=bn_momentum) conv2 = get_conv(name=name + '_conv2', data=conv1, num_filter=num_filter, kernel=(3, 3), stride=(1, 1), pad=(1, 1), with_relu=False, bn_momentum=bn_momentum) if dim_match: shortcut = data else: # Like http://ethereon.github.io/netscope/#/gist/db945b393d40bfa26006 # Test accuracy 0.922+ on CIFAR10 with 56 layers # shortcut = get_conv( # name=name + '_proj', # data=data, # num_filter=num_filter, # kernel=(1, 1), # stride=(2, 2), # pad=(0, 0), # with_relu=False, # bn_momentum=bn_momentum # ) # Type A shortcut # Note we use kernel (2, 2) rather than (1, 1) and a custom initializer # in train_cifar10_resnet.py # Test accuracy 0.918 on CIFAR10 with 56 layers and kernel (1, 1) # TODO(Answeror): Don't know why (1, 1) got much lower accuracy shortcut = mxc.Convolution(name=name + '_proj', data=data, num_filter=num_filter, kernel=(2, 2), stride=(2, 2), pad=(0, 0), no_bias=True) # Same with above, but ugly # Mxnet don't have nn.Padding as that in # https://github.com/gcr/torch-residual-networks/blob/master/residual-layers.lua # shortcut = mxc.Pooling( # data=data, # name=name + '_pool', # kernel=(2, 2), # stride=(2, 2), # pool_type='avg' # ) # shortcut = mxc.Concat( # shortcut, # mxc.minimum(shortcut + 1, 0), # num_args=2 # ) fused = mxc.Add(name=name + '_fused', bottoms=[shortcut, conv2]) return mxc.Activation(name=name + '_relu', data=fused, act_type='relu')
def residual_unit(data, num_filter, stride, dim_match, name, bottle_neck=True, bn_mom=0.9, workspace=512, memonger=False): """Return ResNet Unit symbol for building ResNet Parameters ---------- data : str Input data num_filter : int Number of output channels bnf : int Bottle neck channels factor with regard to num_filter stride : tupe Stride used in convolution dim_match : Boolen True means channel number between input and output is the same, otherwise means differ name : str Base name of the operators workspace : int Workspace used in convolution operator """ if bottle_neck: # the same as https://github.com/facebook/fb.resnet.torch#notes, a bit difference with origin paper bn1 = mxc.BatchNorm(data=data, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn1') act1 = mxc.Activation(data=bn1, act_type='relu', name=name + '_relu1') conv1 = mxc.Convolution(data=act1, num_filter=int(num_filter*0.25), kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, workspace=workspace, name=name + '_conv1') bn2 = mxc.BatchNorm(data=conv1, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn2') act2 = mxc.Activation(data=bn2, act_type='relu', name=name + '_relu2') conv2 = mxc.Convolution(data=act2, num_filter=int(num_filter*0.25), kernel=(3,3), stride=stride, pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv2') bn3 = mxc.BatchNorm(data=conv2, fix_gamma=False, eps=2e-5, momentum=bn_mom, name=name + '_bn3') act3 = mxc.Activation(data=bn3, act_type='relu', name=name + '_relu3') conv3 = mxc.Convolution(data=act3, num_filter=num_filter, kernel=(1,1), stride=(1,1), pad=(0,0), no_bias=True, workspace=workspace, name=name + '_conv3') if dim_match: shortcut = data else: shortcut = mxc.Convolution(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, workspace=workspace, name=name+'_sc') if memonger: shortcut._set_attr(mirror_stage='True') fused = mxc.Add( name = name + '_fused', bottoms = [conv3, shortcut]) return fused else: bn1 = mxc.BatchNorm(data=data, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn1') act1 = mxc.Activation(data=bn1, act_type='relu', name=name + '_relu1') conv1 = mxc.Convolution(data=act1, num_filter=num_filter, kernel=(3,3), stride=stride, pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv1') bn2 = mxc.BatchNorm(data=conv1, fix_gamma=False, momentum=bn_mom, eps=2e-5, name=name + '_bn2') act2 = mxc.Activation(data=bn2, act_type='relu', name=name + '_relu2') conv2 = mxc.Convolution(data=act2, num_filter=num_filter, kernel=(3,3), stride=(1,1), pad=(1,1), no_bias=True, workspace=workspace, name=name + '_conv2') if dim_match: shortcut = data else: shortcut = mxc.Convolution(data=act1, num_filter=num_filter, kernel=(1,1), stride=stride, no_bias=True, workspace=workspace, name=name+'_sc') if memonger: shortcut._set_attr(mirror_stage='True') fused = mxc.Add( name = name + '_fused', bottoms = [conv2, shortcut]) return fused
def Conv(data, num_filter, kernel=(1, 1), stride=(1, 1), pad=(0, 0), name=None, suffix=''): conv = mxc.Convolution(data=data, num_filter=num_filter, kernel=kernel, stride=stride, pad=pad, name='%s%s_conv2d' %(name, suffix)) bn = mxc.BatchNorm(data=conv, name='%s%s_batchnorm' %(name, suffix)) act = mxc.Activation(data=bn, act_type='relu', name='%s%s_relu' %(name, suffix)) return act