def __init__(self,
in_channels,
out_channels,
kernel_size=1,
stride=1,
padding=0,
dilation=1,
groups=1,
bias=False):
"""
:param in_channels:
:param out_channels:
:param kernel_size:
:param stride:
:param padding:
:param dilation:
:param groups:
:param bias:
"""
kernel_size = _pair(kernel_size)
stride = _pair(stride)
padding = _pair(padding)
dilation = _pair(dilation)
super(GatedSpatialConv2d,
self).__init__(in_channels, out_channels,
kernel_size, stride, padding, dilation, False,
_pair(0), groups, bias, 'zeros')
self._gate_conv = nn.Sequential(
mynn.Norm2d(in_channels + 1),
nn.Conv2d(in_channels + 1, in_channels + 1, 1), nn.ReLU(),
nn.Conv2d(in_channels + 1, 1, 1), mynn.Norm2d(1), nn.Sigmoid())
def __init__(self,
inplanes,
planes,
groups,
reduction,
stride=1,
downsample=None,
base_width=4):
super(SEResNeXtBottleneck, self).__init__()
width = math.floor(planes * (base_width / 64)) * groups
self.conv1 = nn.Conv2d(inplanes,
width,
kernel_size=1,
bias=False,
stride=1)
self.bn1 = mynn.Norm2d(width)
self.conv2 = nn.Conv2d(width,
width,
kernel_size=3,
stride=stride,
padding=1,
groups=groups,
bias=False)
self.bn2 = mynn.Norm2d(width)
self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False)
self.bn3 = mynn.Norm2d(planes * 4)
self.relu = nn.ReLU(inplace=True)
self.se_module = SEModule(planes * 4, reduction=reduction)
self.downsample = downsample
self.stride = stride
def __init__(self,
inplanes,
planes,
groups,
reduction,
stride=1,
downsample=None):
super(SEResNetBottleneck, self).__init__()
self.conv1 = nn.Conv2d(inplanes,
planes,
kernel_size=1,
bias=False,
stride=stride)
self.bn1 = mynn.Norm2d(planes)
self.conv2 = nn.Conv2d(planes,
planes,
kernel_size=3,
padding=1,
groups=groups,
bias=False)
self.bn2 = mynn.Norm2d(planes)
self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False)
self.bn3 = mynn.Norm2d(planes * 4)
self.relu = nn.ReLU(inplace=True)
self.se_module = SEModule(planes * 4, reduction=reduction)
self.downsample = downsample
self.stride = stride
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn1 = mynn.Norm2d(planes)
self.relu = nn.ReLU(inplace=True)
self.conv2 = conv3x3(planes, planes)
self.bn2 = mynn.Norm2d(planes)
self.downsample = downsample
self.stride = stride
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(Bottleneck, self).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
self.bn1 = mynn.Norm2d(planes)
self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride,
padding=1, bias=False)
self.bn2 = mynn.Norm2d(planes)
self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False)
self.bn3 = mynn.Norm2d(planes * self.expansion)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.stride = stride
def __init__(self, inplanes, planes, stride=1, downsample=None, iw=0):
super(Bottleneck, self).__init__()
self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False)
self.bn1 = mynn.Norm2d(planes)
self.conv2 = nn.Conv2d(planes,
planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False)
self.bn2 = mynn.Norm2d(planes)
self.conv3 = nn.Conv2d(planes,
planes * self.expansion,
kernel_size=1,
bias=False)
self.bn3 = mynn.Norm2d(planes * self.expansion)
self.downsample = downsample
self.stride = stride
self.iw = iw
if self.iw == 1:
self.instance_norm_layer = InstanceWhitening(planes *
self.expansion)
self.relu = nn.ReLU(inplace=False)
elif self.iw == 2:
self.instance_norm_layer = InstanceWhitening(planes *
self.expansion)
self.relu = nn.ReLU(inplace=False)
elif self.iw == 3:
self.instance_norm_layer = nn.InstanceNorm2d(planes *
self.expansion,
affine=False)
self.relu = nn.ReLU(inplace=True)
elif self.iw == 4:
self.instance_norm_layer = nn.InstanceNorm2d(planes *
self.expansion,
affine=True)
self.relu = nn.ReLU(inplace=True)
elif self.iw == 5:
self.instance_norm_layer = SyncSwitchWhiten2d(planes *
self.expansion,
num_pergroup=16,
sw_type=2,
T=5,
tie_weight=False,
eps=1e-5,
momentum=0.99,
affine=True)
self.relu = nn.ReLU(inplace=True)
else:
self.relu = nn.ReLU(inplace=True)
def __init__(self, block, layers, num_classes=1000):
# self.inplanes = 64
self.inplanes = 128
super(ResNet3X3, self).__init__()
# self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3,
# bias=False)
# self.bn1 = mynn.Norm2d(64)
# self.relu = nn.ReLU(inplace=True)
self.conv1 = nn.Conv2d(3,
64,
kernel_size=3,
stride=2,
padding=1,
bias=False)
self.bn1 = mynn.Norm2d(64)
self.relu1 = nn.ReLU(inplace=True)
self.conv2 = nn.Conv2d(64,
64,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.bn2 = mynn.Norm2d(64)
self.relu2 = nn.ReLU(inplace=True)
self.conv3 = nn.Conv2d(64,
128,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.bn3 = mynn.Norm2d(self.inplanes)
self.relu3 = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
self.avgpool = nn.AvgPool2d(7, stride=1)
self.fc = nn.Linear(512 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d) or isinstance(
m, nn.SyncBatchNorm):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
def _make_layer(self,
block,
planes,
blocks,
groups,
reduction,
stride=1,
downsample_kernel_size=1,
downsample_padding=0):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.Conv2d(self.inplanes,
planes * block.expansion,
kernel_size=downsample_kernel_size,
stride=stride,
padding=downsample_padding,
bias=False),
mynn.Norm2d(planes * block.expansion),
)
layers = []
layers.append(
block(self.inplanes, planes, groups, reduction, stride,
downsample))
self.inplanes = planes * block.expansion
for index in range(1, blocks):
layers.append(block(self.inplanes, planes, groups, reduction))
logging.info("Layer %s Assigned", str(index))
return nn.Sequential(*layers)
def __init__(self, inplanes, planes, stride=1, downsample=None):
super(BasicBlock, self).__init__()
self.conv1 = conv3x3(inplanes, planes, stride)
self.bn1 = mynn.Norm2d(planes)
self.relu = nn.ReLU(inplace=True)
self.conv2 = conv3x3(planes, planes)
self.bn2 = mynn.Norm2d(planes)
self.downsample = downsample
self.stride = stride
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
def _make_layer(self, block, planes, blocks, stride=1):
downsample = None
if stride != 1 or self.inplanes != planes * block.expansion:
downsample = nn.Sequential(
nn.Conv2d(self.inplanes, planes * block.expansion,
kernel_size=1, stride=stride, bias=False),
mynn.Norm2d(planes * block.expansion),
)
layers = []
layers.append(block(self.inplanes, planes, stride, downsample))
self.inplanes = planes * block.expansion
for index in range(1, blocks):
layers.append(block(self.inplanes, planes))
return nn.Sequential(*layers)
def __init__(self,
in_channels,
out_channels,
kernel_size=1,
stride=1,
padding=0,
dilation=1,
groups=1,
bias=False):
"""
:param in_channels:
:param out_channels:
:param kernel_size:
:param stride:
:param padding:
:param dilation:
:param groups:
:param bias:
"""
kernel_size = _pair(kernel_size)
stride = _pair(stride)
padding = _pair(padding)
dilation = _pair(dilation)
super(HighFrequencyGatedSpatialConv2d,
self).__init__(in_channels, out_channels, kernel_size, stride,
padding, dilation, False, _pair(0), groups, bias)
self._gate_conv = nn.Sequential(
mynn.Norm2d(in_channels + 1),
nn.Conv2d(in_channels + 1, in_channels + 1, 1), nn.ReLU(),
nn.Conv2d(in_channels + 1, 1, 1), mynn.Norm2d(1), nn.Sigmoid())
kernel_size = 7
sigma = 3
x_cord = torch.arange(kernel_size).float()
x_grid = x_cord.repeat(kernel_size).view(kernel_size,
kernel_size).float()
y_grid = x_grid.t().float()
xy_grid = torch.stack([x_grid, y_grid], dim=-1).float()
mean = (kernel_size - 1) / 2.
variance = sigma**2.
gaussian_kernel = (1./(2.*math.pi*variance)) *\
torch.exp(
-torch.sum((xy_grid - mean)**2., dim=-1) /\
(2*variance)
)
gaussian_kernel = gaussian_kernel / torch.sum(gaussian_kernel)
gaussian_kernel = gaussian_kernel.view(1, 1, kernel_size, kernel_size)
gaussian_kernel = gaussian_kernel.repeat(in_channels, 1, 1, 1)
self.gaussian_filter = nn.Conv2d(in_channels=in_channels,
out_channels=in_channels,
padding=3,
kernel_size=kernel_size,
groups=in_channels,
bias=False)
self.gaussian_filter.weight.data = gaussian_kernel
self.gaussian_filter.weight.requires_grad = False
self.cw = nn.Conv2d(in_channels * 2, in_channels, 1)
self.procdog = nn.Sequential(nn.Conv2d(in_channels, in_channels, 1),
mynn.Norm2d(in_channels), nn.Sigmoid())
def bnrelu(channels):
return nn.Sequential(mynn.Norm2d(channels), nn.ReLU(inplace=True))
def __init__(self, block, layers, groups, reduction, dropout_p=0.2,
inplanes=128, input_3x3=True, downsample_kernel_size=3,
downsample_padding=1, num_classes=1000):
"""
Parameters
----------
block (nn.Module): Bottleneck class.
- For SENet154: SEBottleneck
- For SE-ResNet models: SEResNetBottleneck
- For SE-ResNeXt models: SEResNeXtBottleneck
layers (list of ints): Number of residual blocks for 4 layers of the
network (layer1...layer4).
groups (int): Number of groups for the 3x3 convolution in each
bottleneck block.
- For SENet154: 64
- For SE-ResNet models: 1
- For SE-ResNeXt models: 32
reduction (int): Reduction ratio for Squeeze-and-Excitation modules.
- For all models: 16
dropout_p (float or None): Drop probability for the Dropout layer.
If `None` the Dropout layer is not used.
- For SENet154: 0.2
- For SE-ResNet models: None
- For SE-ResNeXt models: None
inplanes (int): Number of input channels for layer1.
- For SENet154: 128
- For SE-ResNet models: 64
- For SE-ResNeXt models: 64
input_3x3 (bool): If `True`, use three 3x3 convolutions instead of
a single 7x7 convolution in layer0.
- For SENet154: True
- For SE-ResNet models: False
- For SE-ResNeXt models: False
downsample_kernel_size (int): Kernel size for downsampling convolutions
in layer2, layer3 and layer4.
- For SENet154: 3
- For SE-ResNet models: 1
- For SE-ResNeXt models: 1
downsample_padding (int): Padding for downsampling convolutions in
layer2, layer3 and layer4.
- For SENet154: 1
- For SE-ResNet models: 0
- For SE-ResNeXt models: 0
num_classes (int): Number of outputs in `last_linear` layer.
- For all models: 1000
"""
super(SENet, self).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [
('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1,
bias=False)),
('bn1', mynn.Norm2d(64)),
('relu1', nn.ReLU(inplace=True)),
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)),
('bn2', mynn.Norm2d(64)),
('relu2', nn.ReLU(inplace=True)),
('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1,
bias=False)),
('bn3', mynn.Norm2d(inplanes)),
('relu3', nn.ReLU(inplace=True)),
]
else:
layer0_modules = [
('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2,
padding=3, bias=False)),
('bn1', mynn.Norm2d(inplanes)),
('relu1', nn.ReLU(inplace=True)),
]
# To preserve compatibility with Caffe weights `ceil_mode=True`
# is used instead of `padding=1`.
layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2,
ceil_mode=True)))
self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(
block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0
)
self.layer2 = self._make_layer(
block,
planes=128,
blocks=layers[1],
stride=2,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding
)
self.layer3 = self._make_layer(
block,
planes=256,
blocks=layers[2],
stride=1,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding
)
self.layer4 = self._make_layer(
block,
planes=512,
blocks=layers[3],
stride=1,
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding
)
self.avg_pool = nn.AvgPool2d(7, stride=1)
self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
def bnrelu(channels):
"""
Single Layer BN and Relui
"""
return nn.Sequential(mynn.Norm2d(channels), nn.ReLU(inplace=True))
def __init__(self, block, layers, wt_layer=None, num_classes=1000):
self.inplanes = 64
# self.inplanes = 128
super(ResNet, self).__init__()
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False)
if wt_layer[2] == 1:
self.bn1 = InstanceWhitening(64)
self.relu = nn.ReLU(inplace=False)
elif wt_layer[2] == 2:
self.bn1 = InstanceWhitening(64)
self.relu = nn.ReLU(inplace=False)
elif wt_layer[2] == 3:
self.bn1 = nn.InstanceNorm2d(64, affine=False)
self.relu = nn.ReLU(inplace=True)
elif wt_layer[2] == 4:
self.bn1 = nn.InstanceNorm2d(64, affine=True)
self.relu = nn.ReLU(inplace=True)
elif wt_layer[2] == 5:
self.bn1 = SyncSwitchWhiten2d(self.inplanes,
num_pergroup=16,
sw_type=2,
T=5,
tie_weight=False,
eps=1e-5,
momentum=0.99,
affine=True)
self.relu = nn.ReLU(inplace=True)
else:
self.bn1 = mynn.Norm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block,
64,
layers[0],
wt_layer=wt_layer[3])
self.layer2 = self._make_layer(block,
128,
layers[1],
stride=2,
wt_layer=wt_layer[4])
self.layer3 = self._make_layer(block,
256,
layers[2],
stride=2,
wt_layer=wt_layer[5])
self.layer4 = self._make_layer(block,
512,
layers[3],
stride=2,
wt_layer=wt_layer[6])
self.avgpool = nn.AvgPool2d(7, stride=1)
self.fc = nn.Linear(512 * block.expansion, num_classes)
self.wt_layer = wt_layer
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d) or isinstance(
m, nn.SyncBatchNorm):
if m.weight is not None:
nn.init.constant_(m.weight, 1)
if m.bias is not None:
nn.init.constant_(m.bias, 0)
