def __init__(self,
inplanes,
planes,
baseWidth,
cardinality,
stride=1,
downsample=None,
part=0,
zero_fixed_part=False,
do_init=False):
""" Constructor
Args:
inplanes: input channel dimensionality
planes: output channel dimensionality
baseWidth: base width.
cardinality: num of convolution groups.
stride: conv stride. Replaces pooling layer.
"""
super(Bottleneck, self).__init__()
#print '----------------->',do_init
D = int(math.floor(planes * (baseWidth / 64)))
C = cardinality
self.conv1 = Conv2D_partial(
nn.Conv2d(inplanes,
D * C,
kernel_size=1,
stride=1,
padding=0,
bias=False), part, zero_fixed_part, do_init)
self.bn1 = nn.BatchNorm2d(D * C)
self.conv2 = Conv2D_partial(
nn.Conv2d(D * C,
D * C,
kernel_size=3,
stride=stride,
padding=1,
groups=C,
bias=False), part, zero_fixed_part, do_init)
self.bn2 = nn.BatchNorm2d(D * C)
self.conv3 = Conv2D_partial(
nn.Conv2d(D * C,
planes * 4,
kernel_size=1,
stride=1,
padding=0,
bias=False), part, zero_fixed_part, do_init)
self.bn3 = nn.BatchNorm2d(planes * 4)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
def __init__(self, depth, num_classes, widen_factor=1, dropRate=0.0, lastpool=8,
part=1.0,zero_fixed_part=False,do_init=True):
super(WideResNet_partial, self).__init__()
nChannels = [16, 16 * widen_factor, 32 * widen_factor, 64 * widen_factor]
assert ((depth - 4) % 6 == 0)
n = (depth - 4) / 6
block = BasicBlock
# 1st conv before any network block
self.conv1 = Conv2D_partial(nn.Conv2d(3, nChannels[0], kernel_size=3, stride=1,
padding=1, bias=False),part,zero_fixed_part,do_init)
# 1st block
self.block1 = NetworkBlock(n, nChannels[0], nChannels[1], block, 2, dropRate,part,zero_fixed_part,do_init)
# 2nd block
self.block2 = NetworkBlock(n, nChannels[1], nChannels[2], block, 2, dropRate,part,zero_fixed_part,do_init)
# 3rd block
self.block3 = NetworkBlock(n, nChannels[2], nChannels[3], block, 2, dropRate,part,zero_fixed_part,do_init)
# global average pooling and classifier
self.bn1 = nn.BatchNorm2d(nChannels[3], affine=myAffine)
self.relu = nn.ReLU(inplace=True)
self.fc = nn.Linear(nChannels[3], num_classes)
self.nChannels = nChannels[3]
self.lastpool = lastpool
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d) and m.affine:
m.weight.data.fill_(1)
m.bias.data.zero_()
elif isinstance(m, nn.Linear):
m.bias.data.zero_()
def __init__(self, in_planes, out_planes, stride, dropRate=0.0,part=1.0,zero_fixed_part=False,do_init=True):
super(BasicBlock, self).__init__()
self.bn1 = nn.BatchNorm2d(in_planes, affine=myAffine)
self.relu1 = nn.ReLU(inplace=True)
self.conv1 = Conv2D_partial(nn.Conv2d(in_planes, out_planes, kernel_size=3, stride=stride,
padding=1, bias=False),part,zero_fixed_part,do_init)
self.bn2 = nn.BatchNorm2d(out_planes, affine=myAffine)
self.relu2 = nn.ReLU(inplace=True)
self.conv2 = Conv2D_partial(nn.Conv2d(out_planes, out_planes, kernel_size=3, stride=1,
padding=1, bias=False),part,zero_fixed_part,do_init)
self.droprate = dropRate
self.equalInOut = (in_planes == out_planes)
self.convShortcut = (not self.equalInOut) and \
Conv2D_partial(nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=stride,
padding=0, bias=False),part,zero_fixed_part,do_init)\
or None
def __init__(self,
inplanes,
expansion=4,
growthRate=12,
dropRate=0,
part=1.0,
zero_fixed_part=False,
do_init=False,
split_dim=0):
super(Bottleneck, self).__init__()
planes = expansion * growthRate
self.bn1 = nn.BatchNorm2d(inplanes)
self.conv1 = Conv2D_partial(
nn.Conv2d(inplanes, planes, kernel_size=1, bias=False), part,
zero_fixed_part, do_init, split_dim)
self.bn2 = nn.BatchNorm2d(planes)
self.conv2 = Conv2D_partial(
nn.Conv2d(planes, growthRate, kernel_size=3, padding=1,
bias=False), part, zero_fixed_part, do_init, split_dim)
self.relu = nn.ReLU(inplace=True)
self.dropRate = dropRate
def __init__(self,
depth=22,
block=Bottleneck,
dropRate=0,
num_classes=10,
growthRate=12,
compressionRate=2,
part=1.0,
zero_fixed_part=False,
do_init=False,
split_dim=0):
super(DenseNet_partial, self).__init__()
assert (depth - 4) % 3 == 0, 'depth should be 3n+4'
n = (depth - 4) / 3 if block == BasicBlock else (depth - 4) // 6
self.growthRate = growthRate
self.dropRate = dropRate
# self.inplanes is a global variable used across multiple
# helper functions
self.inplanes = growthRate * 2
#print 'BAH'
self.conv1 = Conv2D_partial(
nn.Conv2d(3, self.inplanes, kernel_size=3, padding=1, bias=False),
part, zero_fixed_part, do_init, split_dim)
#print 'BOO'
self.dense1 = self._make_denseblock(block, n, part, zero_fixed_part,
do_init, split_dim)
self.trans1 = self._make_transition(compressionRate, part,
zero_fixed_part, do_init,
split_dim)
self.dense2 = self._make_denseblock(block, n, part, zero_fixed_part,
do_init, split_dim)
self.trans2 = self._make_transition(compressionRate, part,
zero_fixed_part, do_init,
split_dim)
self.dense3 = self._make_denseblock(block, n, part, zero_fixed_part,
do_init, split_dim)
self.bn = nn.BatchNorm2d(self.inplanes)
self.relu = nn.ReLU(inplace=True)
self.avgpool = nn.AvgPool2d(8)
self.fc = nn.Linear(self.inplanes, num_classes)
# Weight initialization
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def __init__(self,
inplanes,
outplanes,
part=1.0,
zero_fixed_part=False,
do_init=False,
split_dim=0):
super(Transition, self).__init__()
self.bn1 = nn.BatchNorm2d(inplanes)
self.conv1 = Conv2D_partial(
nn.Conv2d(inplanes, outplanes, kernel_size=1, bias=False), part,
zero_fixed_part, do_init, split_dim)
self.relu = nn.ReLU(inplace=True)
def __init__(self,
baseWidth,
cardinality,
layers,
num_classes,
part=1.0,
zero_fixed_part=False,
do_init=False):
""" Constructor
Args:
baseWidth: baseWidth for ResNeXt.
cardinality: number of convolution groups.
layers: config of layers, e.g., [3, 4, 6, 3]
num_classes: number of classes
"""
super(ResNeXt_partial, self).__init__()
block = Bottleneck
self.cardinality = cardinality
self.baseWidth = baseWidth
self.num_classes = num_classes
self.inplanes = 64
self.output_size = 64
self.conv1 = Conv2D_partial(nn.Conv2d(3, 64, 7, 2, 3, bias=False),
part, zero_fixed_part, do_init)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool1 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
#print '-----------------',do_init,'-----------'
self.layer1 = self._make_layer(block, 64, layers[0], 1, part,
zero_fixed_part, do_init)
self.layer2 = self._make_layer(block, 128, layers[1], 2, part,
zero_fixed_part, do_init)
self.layer3 = self._make_layer(block, 256, layers[2], 2, part,
zero_fixed_part, do_init)
self.layer4 = self._make_layer(block, 512, layers[3], 2, part,
zero_fixed_part, do_init)
self.avgpool = nn.AvgPool2d(2)
#self.fc = nn.Linear(512 * block.expansion, num_classes)
self.fc = nn.Linear(2048, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def __init__(self, num_classes=10, part=1.0, zero_fixed_part=False):
super(AlexNet_partial, self).__init__()
self.features = nn.Sequential(
Conv2D_partial(
nn.Conv2d(3, 64, kernel_size=11, stride=4, padding=5), part,
zero_fixed_part),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
Conv2D_partial(nn.Conv2d(64, 192, kernel_size=5, padding=2), part,
zero_fixed_part),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
Conv2D_partial(nn.Conv2d(192, 384, kernel_size=3, padding=1), part,
zero_fixed_part),
nn.ReLU(inplace=True),
Conv2D_partial(nn.Conv2d(384, 256, kernel_size=3, padding=1), part,
zero_fixed_part),
nn.ReLU(inplace=True),
Conv2D_partial(nn.Conv2d(256, 256, kernel_size=3, padding=1), part,
zero_fixed_part),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
)
self.classifier = nn.Linear(256, num_classes)
def __init__(self,
inplanes,
squeeze_planes,
expand1x1_planes,
expand3x3_planes,
part,
zero_fixed_part=False,
do_init=False):
super(Fire, self).__init__()
self.inplanes = inplanes
self.squeeze = Conv2D_partial(
nn.Conv2d(inplanes, squeeze_planes, kernel_size=1), part,
zero_fixed_part, do_init)
self.squeeze_activation = nn.ReLU(inplace=True)
self.expand1x1 = Conv2D_partial(
nn.Conv2d(squeeze_planes, expand1x1_planes, kernel_size=1), part,
zero_fixed_part, do_init)
self.expand1x1_activation = nn.ReLU(inplace=True)
self.expand3x3 = Conv2D_partial(
nn.Conv2d(squeeze_planes,
expand3x3_planes,
kernel_size=3,
padding=1), part, zero_fixed_part, do_init)
self.expand3x3_activation = nn.ReLU(inplace=True)
def make_layers(cfg, batch_norm=False,part=1.0,zero_fixed_part=False):
print('INSIDE MAKE LAYERS,PART=',part)
layers = []
in_channels = 3
for v in cfg:
if v == 'M':
layers += [nn.MaxPool2d(kernel_size=2, stride=2)]
else:
conv2d = Conv2D_partial(nn.Conv2d(in_channels, v, kernel_size=3, padding=1),part,zero_fixed_part)
if batch_norm:
layers += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)]
else:
layers += [conv2d, nn.ReLU(inplace=True)]
in_channels = v
return nn.Sequential(*layers)
def _make_layer(self,
block,
planes,
blocks,
stride=1,
part=0,
zero_fixed_part=False,
do_init=False):
""" Stack n bottleneck modules where n is inferred from the depth of the network.
Args:
block: block type used to construct ResNext
planes: number of output channels (need to multiply by block.expansion)
blocks: number of blocks to be built
stride: factor to reduce the spatial dimensionality in the first bottleneck of the block.
Returns: a Module consisting of n sequential bottlenecks.
"""
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
Conv2D_partial(
nn.Conv2d(self.inplanes,
planes * block.expansion,
kernel_size=1,
stride=stride,
bias=False), part, zero_fixed_part, do_init),
nn.BatchNorm2d(planes * block.expansion),
)
layers = []
layers.append(
block(self.inplanes, planes, self.baseWidth, self.cardinality,
stride, downsample, part, zero_fixed_part, do_init))
self.inplanes = planes * block.expansion
for i in range(1, blocks):
layers.append(
block(self.inplanes, planes, self.baseWidth, self.cardinality,
1, None, part, zero_fixed_part, do_init))
return nn.Sequential(*layers)
def __init__(self,
version=1.0,
num_classes=1000,
part=1.0,
zero_fixed_part=False,
do_init=False):
super(SqueezeNet_partial, self).__init__()
if version not in [1.0, 1.1]:
raise ValueError("Unsupported SqueezeNet version {version}:"
"1.0 or 1.1 expected".format(version=version))
self.num_classes = num_classes
if version == 1.0:
self.features = nn.Sequential(
Conv2D_partial(nn.Conv2d(3, 96, kernel_size=7, stride=2), part,
zero_fixed_part, do_init),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),
Fire(96, 16, 64, 64, part, zero_fixed_part, do_init),
Fire(128, 16, 64, 64, part, zero_fixed_part, do_init),
Fire(128, 32, 128, 128, part, zero_fixed_part, do_init),
nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),
Fire(256, 32, 128, 128, part, zero_fixed_part, do_init),
Fire(256, 48, 192, 192, part, zero_fixed_part, do_init),
Fire(384, 48, 192, 192, part, zero_fixed_part, do_init),
Fire(384, 64, 256, 256, part, zero_fixed_part, do_init),
nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),
Fire(512, 64, 256, 256, part, zero_fixed_part),
)
else:
self.features = nn.Sequential(
Conv2D_partial(nn.Conv2d(3, 64, kernel_size=3, stride=2), part,
zero_fixed_part, do_init),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),
Fire(64, 16, 64, 64, part, zero_fixed_part, do_init),
Fire(128, 16, 64, 64, part, zero_fixed_part, do_init),
nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),
Fire(128, 32, 128, 128, part, zero_fixed_part, do_init),
Fire(256, 32, 128, 128, part, zero_fixed_part, do_init),
nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True),
Fire(256, 48, 192, 192, part, zero_fixed_part, do_init),
Fire(384, 48, 192, 192, part, zero_fixed_part, do_init),
Fire(384, 64, 256, 256, part, zero_fixed_part, do_init),
Fire(512, 64, 256, 256, part, zero_fixed_part, do_init),
)
# Final convolution is initialized differently form the rest
final_conv = nn.Conv2d(512, self.num_classes, kernel_size=1)
self.classifier = nn.Sequential(
nn.Dropout(p=0.5), final_conv
#nn.ReLU(inplace=True)
#nn.AvgPool2d(13, stride=1)
)
for m in self.modules():
if isinstance(m, nn.Conv2d):
if m is final_conv:
init.normal(m.weight.data, mean=0.0, std=0.01)
else:
init.kaiming_uniform(m.weight.data)
if m.bias is not None:
m.bias.data.zero_()