def __init__(self,
inplanes,
planes,
stride=1,
downsample=None,
reduction=16,
t=1):
super(Bottleneck, self).__init__()
self.bn1 = RWBatchNorm2d(inplanes, ratio=1)
self.conv1 = RWConv2d(inplanes,
planes,
kernel_size=1,
bias=False,
ratio=[1, 1])
self.bn2 = RWBatchNorm2d(planes)
self.conv2 = RWConv2d(planes,
planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False,
groups=1)
self.bn3 = RWBatchNorm2d(planes)
self.conv3 = RWConv2d(planes,
planes * Bottleneck.outchannel_ratio,
kernel_size=1,
bias=False,
ratio=[1.0, 1.0])
self.bn4 = RWBatchNorm2d(planes * Bottleneck.outchannel_ratio, ratio=1)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def __init__(self, in_planes, out_planes, stride, dropRate=0.0):
super(BasicBlock, self).__init__()
self.bn1 = RWBatchNorm2d(in_planes)
self.relu1 = nn.ReLU(inplace=True)
self.conv1 = RWConv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False)
self.bn2 = RWBatchNorm2d(out_planes)
self.relu2 = nn.ReLU(inplace=True)
self.conv2 = RWConv2d(out_planes, out_planes, kernel_size=3, stride=1,
padding=1, bias=False)
self.droprate = dropRate
self.equalInOut = (in_planes == out_planes)
self.convShortcut = (not self.equalInOut) and RWConv2d(in_planes, out_planes, kernel_size=1, stride=stride,
padding=0, bias=False) or None
def __init__(self, depth, num_classes, widen_factor=1, dropRate=0.0):
super(WideResNet_randwidth, self).__init__()
nChannels = [16, 16*widen_factor, 32*widen_factor, 64*widen_factor]
assert((depth - 4) % 6 == 0)
n = (depth - 4) / 6
block = BasicBlock
# 1st conv before any network block
self.conv1 = RWConv2d(3, nChannels[0], kernel_size=3, stride=1,
padding=1, bias=False, us=[False, True])
# 1st block
self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 1, dropRate)
# 2nd block
self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate)
# 3rd block
self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate)
# global average pooling and classifier
self.bn1 = RWBatchNorm2d(nChannels[3])
self.relu = nn.ReLU(inplace=True)
self.fc = RWLinear(nChannels[3], num_classes, us=[True, False])
self.nChannels = nChannels[3]
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d):
if m.affine:
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
m.bias.data.zero_()
def __init__(self, inp, outp, stride, tmp_ratio=1.0):
super(Block, self).__init__()
assert stride in [1, 2]
# midp = [i // 4 for i in outp]
midp = make_divisible(outp // 4)
expand_ratio = 0.25
layers = [
RWConv2d(inp,
midp,
1,
1,
0,
bias=False,
ratio=[tmp_ratio, expand_ratio]),
RWBatchNorm2d(midp, ratio=expand_ratio),
nn.ReLU(inplace=True),
RWConv2d(midp,
midp,
3,
stride,
1,
bias=False,
ratio=[expand_ratio, expand_ratio]),
RWBatchNorm2d(midp, ratio=expand_ratio),
nn.ReLU(inplace=True),
RWConv2d(midp, outp, 1, 1, 0, bias=False, ratio=[expand_ratio, 1]),
RWBatchNorm2d(outp),
]
self.body = nn.Sequential(*layers)
self.residual_connection = stride == 1 and inp == outp
if not self.residual_connection:
self.shortcut = nn.Sequential(
RWConv2d(inp,
outp,
1,
stride=stride,
bias=False,
ratio=[tmp_ratio, 1]),
RWBatchNorm2d(outp),
)
self.post_relu = nn.ReLU(inplace=True)
def __init__(self, depth=50, num_classes=1000, input_size=224):
super(Model, self).__init__()
self.features = []
# head
assert input_size % 32 == 0
# setting of inverted residual blocks
self.block_setting_dict = {
# : [stage1, stage2, stage3, stage4]
50: [3, 4, 6, 3],
101: [3, 4, 23, 3],
152: [3, 8, 36, 3],
}
init_channel = 64
width_mult = FLAGS.max_width # upper bound
channels = make_divisible(init_channel * width_mult)
self.block_setting = self.block_setting_dict[depth]
feats = [64, 128, 256, 512]
# channels = [
# int(64 * width_mult) for width_mult in FLAGS.width_mult_list]
self.features.append(
nn.Sequential(
RWConv2d(3,
channels,
7,
2,
3,
bias=False,
us=[False, True],
ratio=[1, 0.25]),
RWBatchNorm2d(channels, ratio=0.25),
nn.ReLU(inplace=True),
nn.MaxPool2d(3, 2, 1),
))
# body
for stage_id, n in enumerate(self.block_setting):
outp = make_divisible(feats[stage_id] * width_mult * 4)
# outp = [
# int(feats[stage_id] * width_mult * 4)
# for width_mult in FLAGS.width_mult_list]
for i in range(n):
if i == 0 and stage_id != 0:
self.features.append(Block(channels, outp, 2))
elif i == 0 and stage_id == 0:
self.features.append(
Block(channels, outp, 1, tmp_ratio=0.25))
else:
self.features.append(Block(channels, outp, 1))
channels = outp
avg_pool_size = input_size // 32
self.features.append(nn.AdaptiveAvgPool2d((1, 1)))
# make it nn.Sequential
self.features = nn.Sequential(*self.features)
# classifier
self.outp = channels
self.classifier = nn.Sequential(
RWLinear(self.outp, num_classes, us=[True, False]))
if FLAGS.reset_parameters:
self.reset_parameters()
def __init__(self, dataset, depth, alpha, num_classes, bottleneck=True):
super(PyramidNet_randwidth, self).__init__()
self.dataset = dataset
if self.dataset.startswith('cifar'):
self.inplanes = 16
if bottleneck == True:
n = int((depth - 2) / 9)
block = Bottleneck
else:
n = int((depth - 2) / 6)
block = BasicBlock
self.addrate = alpha / (3 * n * 1.0)
self.input_featuremap_dim = self.inplanes
self.conv1 = RWConv2d(3,
self.input_featuremap_dim,
kernel_size=3,
stride=1,
padding=1,
bias=False,
us=[False, True])
self.bn1 = RWBatchNorm2d(self.input_featuremap_dim)
self.featuremap_dim = self.input_featuremap_dim
self.layer1 = self.pyramidal_make_layer(block, n)
self.layer2 = self.pyramidal_make_layer(block, n, stride=2)
self.layer3 = self.pyramidal_make_layer(block, n, stride=2)
self.final_featuremap_dim = self.input_featuremap_dim
self.bn_final = RWBatchNorm2d(self.final_featuremap_dim)
self.relu_final = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(8)
self.fc = RWLinear(self.final_featuremap_dim,
num_classes,
us=[True, False])
elif dataset == 'imagenet':
blocks = {
18: BasicBlock,
34: BasicBlock,
50: Bottleneck,
101: Bottleneck,
152: Bottleneck,
200: Bottleneck
}
layers = {
18: [2, 2, 2, 2],
34: [3, 4, 6, 3],
50: [3, 4, 6, 3],
101: [3, 4, 23, 3],
152: [3, 8, 36, 3],
200: [3, 24, 36, 3]
}
if layers.get(depth) is None:
if bottleneck == True:
blocks[depth] = Bottleneck
temp_cfg = int((depth - 2) / 12)
else:
blocks[depth] = BasicBlock
temp_cfg = int((depth - 2) / 8)
layers[depth] = [temp_cfg, temp_cfg, temp_cfg, temp_cfg]
print('=> the layer configuration for each stage is set to',
layers[depth])
self.inplanes = 64
self.addrate = alpha / (sum(layers[depth]) * 1.0)
self.input_featuremap_dim = self.inplanes
self.conv1 = nn.Conv2d(3,
self.input_featuremap_dim,
kernel_size=7,
stride=2,
padding=3,
bias=False)
self.bn1 = nn.BatchNorm2d(self.input_featuremap_dim)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.featuremap_dim = self.input_featuremap_dim
self.layer1 = self.pyramidal_make_layer(blocks[depth],
layers[depth][0])
self.layer2 = self.pyramidal_make_layer(blocks[depth],
layers[depth][1],
stride=2)
self.layer3 = self.pyramidal_make_layer(blocks[depth],
layers[depth][2],
stride=2)
self.layer4 = self.pyramidal_make_layer(blocks[depth],
layers[depth][3],
stride=2)
self.final_featuremap_dim = self.input_featuremap_dim
self.bn_final = nn.BatchNorm2d(self.final_featuremap_dim)
self.relu_final = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(7)
self.fc = nn.Linear(self.final_featuremap_dim, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()