def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, padding=0, dilation=1, groups=1, bias=False): """ :param in_channels: :param out_channels: :param kernel_size: :param stride: :param padding: :param dilation: :param groups: :param bias: """ kernel_size = _pair(kernel_size) stride = _pair(stride) padding = _pair(padding) dilation = _pair(dilation) super(GatedSpatialConv2d, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, False, _pair(0), groups, bias, 'zeros') self._gate_conv = nn.Sequential( mynn.Norm2d(in_channels + 1), nn.Conv2d(in_channels + 1, in_channels + 1, 1), nn.ReLU(), nn.Conv2d(in_channels + 1, 1, 1), mynn.Norm2d(1), nn.Sigmoid())
def __init__(self, inplanes, planes, groups, reduction, stride=1, downsample=None, base_width=4): super(SEResNeXtBottleneck, self).__init__() width = math.floor(planes * (base_width / 64)) * groups self.conv1 = nn.Conv2d(inplanes, width, kernel_size=1, bias=False, stride=1) self.bn1 = mynn.Norm2d(width) self.conv2 = nn.Conv2d(width, width, kernel_size=3, stride=stride, padding=1, groups=groups, bias=False) self.bn2 = mynn.Norm2d(width) self.conv3 = nn.Conv2d(width, planes * 4, kernel_size=1, bias=False) self.bn3 = mynn.Norm2d(planes * 4) self.relu = nn.ReLU(inplace=True) self.se_module = SEModule(planes * 4, reduction=reduction) self.downsample = downsample self.stride = stride
def __init__(self, inplanes, planes, groups, reduction, stride=1, downsample=None): super(SEResNetBottleneck, self).__init__() self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False, stride=stride) self.bn1 = mynn.Norm2d(planes) self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, padding=1, groups=groups, bias=False) self.bn2 = mynn.Norm2d(planes) self.conv3 = nn.Conv2d(planes, planes * 4, kernel_size=1, bias=False) self.bn3 = mynn.Norm2d(planes * 4) self.relu = nn.ReLU(inplace=True) self.se_module = SEModule(planes * 4, reduction=reduction) self.downsample = downsample self.stride = stride
def __init__(self, inplanes, planes, stride=1, downsample=None): super(BasicBlock, self).__init__() self.conv1 = conv3x3(inplanes, planes, stride) self.bn1 = mynn.Norm2d(planes) self.relu = nn.ReLU(inplace=True) self.conv2 = conv3x3(planes, planes) self.bn2 = mynn.Norm2d(planes) self.downsample = downsample self.stride = stride
def __init__(self, inplanes, planes, stride=1, downsample=None): super(Bottleneck, self).__init__() self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) self.bn1 = mynn.Norm2d(planes) self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) self.bn2 = mynn.Norm2d(planes) self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False) self.bn3 = mynn.Norm2d(planes * self.expansion) self.relu = nn.ReLU(inplace=True) self.downsample = downsample self.stride = stride
def __init__(self, inplanes, planes, stride=1, downsample=None, iw=0): super(Bottleneck, self).__init__() self.conv1 = nn.Conv2d(inplanes, planes, kernel_size=1, bias=False) self.bn1 = mynn.Norm2d(planes) self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, bias=False) self.bn2 = mynn.Norm2d(planes) self.conv3 = nn.Conv2d(planes, planes * self.expansion, kernel_size=1, bias=False) self.bn3 = mynn.Norm2d(planes * self.expansion) self.downsample = downsample self.stride = stride self.iw = iw if self.iw == 1: self.instance_norm_layer = InstanceWhitening(planes * self.expansion) self.relu = nn.ReLU(inplace=False) elif self.iw == 2: self.instance_norm_layer = InstanceWhitening(planes * self.expansion) self.relu = nn.ReLU(inplace=False) elif self.iw == 3: self.instance_norm_layer = nn.InstanceNorm2d(planes * self.expansion, affine=False) self.relu = nn.ReLU(inplace=True) elif self.iw == 4: self.instance_norm_layer = nn.InstanceNorm2d(planes * self.expansion, affine=True) self.relu = nn.ReLU(inplace=True) elif self.iw == 5: self.instance_norm_layer = SyncSwitchWhiten2d(planes * self.expansion, num_pergroup=16, sw_type=2, T=5, tie_weight=False, eps=1e-5, momentum=0.99, affine=True) self.relu = nn.ReLU(inplace=True) else: self.relu = nn.ReLU(inplace=True)
def __init__(self, block, layers, num_classes=1000): # self.inplanes = 64 self.inplanes = 128 super(ResNet3X3, self).__init__() # self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, # bias=False) # self.bn1 = mynn.Norm2d(64) # self.relu = nn.ReLU(inplace=True) self.conv1 = nn.Conv2d(3, 64, kernel_size=3, stride=2, padding=1, bias=False) self.bn1 = mynn.Norm2d(64) self.relu1 = nn.ReLU(inplace=True) self.conv2 = nn.Conv2d(64, 64, kernel_size=3, stride=1, padding=1, bias=False) self.bn2 = mynn.Norm2d(64) self.relu2 = nn.ReLU(inplace=True) self.conv3 = nn.Conv2d(64, 128, kernel_size=3, stride=1, padding=1, bias=False) self.bn3 = mynn.Norm2d(self.inplanes) self.relu3 = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2) self.layer3 = self._make_layer(block, 256, layers[2], stride=2) self.layer4 = self._make_layer(block, 512, layers[3], stride=2) self.avgpool = nn.AvgPool2d(7, stride=1) self.fc = nn.Linear(512 * block.expansion, num_classes) for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif isinstance(m, nn.BatchNorm2d) or isinstance( m, nn.SyncBatchNorm): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0)
def _make_layer(self, block, planes, blocks, groups, reduction, stride=1, downsample_kernel_size=1, downsample_padding=0): downsample = None if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=downsample_kernel_size, stride=stride, padding=downsample_padding, bias=False), mynn.Norm2d(planes * block.expansion), ) layers = [] layers.append( block(self.inplanes, planes, groups, reduction, stride, downsample)) self.inplanes = planes * block.expansion for index in range(1, blocks): layers.append(block(self.inplanes, planes, groups, reduction)) logging.info("Layer %s Assigned", str(index)) return nn.Sequential(*layers)
def __init__(self, inplanes, planes, stride=1, downsample=None): super(BasicBlock, self).__init__() self.conv1 = conv3x3(inplanes, planes, stride) self.bn1 = mynn.Norm2d(planes) self.relu = nn.ReLU(inplace=True) self.conv2 = conv3x3(planes, planes) self.bn2 = mynn.Norm2d(planes) self.downsample = downsample self.stride = stride for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif isinstance(m, nn.BatchNorm2d): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0)
def _make_layer(self, block, planes, blocks, stride=1): downsample = None if stride != 1 or self.inplanes != planes * block.expansion: downsample = nn.Sequential( nn.Conv2d(self.inplanes, planes * block.expansion, kernel_size=1, stride=stride, bias=False), mynn.Norm2d(planes * block.expansion), ) layers = [] layers.append(block(self.inplanes, planes, stride, downsample)) self.inplanes = planes * block.expansion for index in range(1, blocks): layers.append(block(self.inplanes, planes)) return nn.Sequential(*layers)
def __init__(self, in_channels, out_channels, kernel_size=1, stride=1, padding=0, dilation=1, groups=1, bias=False): """ :param in_channels: :param out_channels: :param kernel_size: :param stride: :param padding: :param dilation: :param groups: :param bias: """ kernel_size = _pair(kernel_size) stride = _pair(stride) padding = _pair(padding) dilation = _pair(dilation) super(HighFrequencyGatedSpatialConv2d, self).__init__(in_channels, out_channels, kernel_size, stride, padding, dilation, False, _pair(0), groups, bias) self._gate_conv = nn.Sequential( mynn.Norm2d(in_channels + 1), nn.Conv2d(in_channels + 1, in_channels + 1, 1), nn.ReLU(), nn.Conv2d(in_channels + 1, 1, 1), mynn.Norm2d(1), nn.Sigmoid()) kernel_size = 7 sigma = 3 x_cord = torch.arange(kernel_size).float() x_grid = x_cord.repeat(kernel_size).view(kernel_size, kernel_size).float() y_grid = x_grid.t().float() xy_grid = torch.stack([x_grid, y_grid], dim=-1).float() mean = (kernel_size - 1) / 2. variance = sigma**2. gaussian_kernel = (1./(2.*math.pi*variance)) *\ torch.exp( -torch.sum((xy_grid - mean)**2., dim=-1) /\ (2*variance) ) gaussian_kernel = gaussian_kernel / torch.sum(gaussian_kernel) gaussian_kernel = gaussian_kernel.view(1, 1, kernel_size, kernel_size) gaussian_kernel = gaussian_kernel.repeat(in_channels, 1, 1, 1) self.gaussian_filter = nn.Conv2d(in_channels=in_channels, out_channels=in_channels, padding=3, kernel_size=kernel_size, groups=in_channels, bias=False) self.gaussian_filter.weight.data = gaussian_kernel self.gaussian_filter.weight.requires_grad = False self.cw = nn.Conv2d(in_channels * 2, in_channels, 1) self.procdog = nn.Sequential(nn.Conv2d(in_channels, in_channels, 1), mynn.Norm2d(in_channels), nn.Sigmoid())
def bnrelu(channels): return nn.Sequential(mynn.Norm2d(channels), nn.ReLU(inplace=True))
def __init__(self, block, layers, groups, reduction, dropout_p=0.2, inplanes=128, input_3x3=True, downsample_kernel_size=3, downsample_padding=1, num_classes=1000): """ Parameters ---------- block (nn.Module): Bottleneck class. - For SENet154: SEBottleneck - For SE-ResNet models: SEResNetBottleneck - For SE-ResNeXt models: SEResNeXtBottleneck layers (list of ints): Number of residual blocks for 4 layers of the network (layer1...layer4). groups (int): Number of groups for the 3x3 convolution in each bottleneck block. - For SENet154: 64 - For SE-ResNet models: 1 - For SE-ResNeXt models: 32 reduction (int): Reduction ratio for Squeeze-and-Excitation modules. - For all models: 16 dropout_p (float or None): Drop probability for the Dropout layer. If `None` the Dropout layer is not used. - For SENet154: 0.2 - For SE-ResNet models: None - For SE-ResNeXt models: None inplanes (int): Number of input channels for layer1. - For SENet154: 128 - For SE-ResNet models: 64 - For SE-ResNeXt models: 64 input_3x3 (bool): If `True`, use three 3x3 convolutions instead of a single 7x7 convolution in layer0. - For SENet154: True - For SE-ResNet models: False - For SE-ResNeXt models: False downsample_kernel_size (int): Kernel size for downsampling convolutions in layer2, layer3 and layer4. - For SENet154: 3 - For SE-ResNet models: 1 - For SE-ResNeXt models: 1 downsample_padding (int): Padding for downsampling convolutions in layer2, layer3 and layer4. - For SENet154: 1 - For SE-ResNet models: 0 - For SE-ResNeXt models: 0 num_classes (int): Number of outputs in `last_linear` layer. - For all models: 1000 """ super(SENet, self).__init__() self.inplanes = inplanes if input_3x3: layer0_modules = [ ('conv1', nn.Conv2d(3, 64, 3, stride=2, padding=1, bias=False)), ('bn1', mynn.Norm2d(64)), ('relu1', nn.ReLU(inplace=True)), ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1, bias=False)), ('bn2', mynn.Norm2d(64)), ('relu2', nn.ReLU(inplace=True)), ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1, bias=False)), ('bn3', mynn.Norm2d(inplanes)), ('relu3', nn.ReLU(inplace=True)), ] else: layer0_modules = [ ('conv1', nn.Conv2d(3, inplanes, kernel_size=7, stride=2, padding=3, bias=False)), ('bn1', mynn.Norm2d(inplanes)), ('relu1', nn.ReLU(inplace=True)), ] # To preserve compatibility with Caffe weights `ceil_mode=True` # is used instead of `padding=1`. layer0_modules.append(('pool', nn.MaxPool2d(3, stride=2, ceil_mode=True))) self.layer0 = nn.Sequential(OrderedDict(layer0_modules)) self.layer1 = self._make_layer( block, planes=64, blocks=layers[0], groups=groups, reduction=reduction, downsample_kernel_size=1, downsample_padding=0 ) self.layer2 = self._make_layer( block, planes=128, blocks=layers[1], stride=2, groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding ) self.layer3 = self._make_layer( block, planes=256, blocks=layers[2], stride=1, groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding ) self.layer4 = self._make_layer( block, planes=512, blocks=layers[3], stride=1, groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding ) self.avg_pool = nn.AvgPool2d(7, stride=1) self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None self.last_linear = nn.Linear(512 * block.expansion, num_classes)
def bnrelu(channels): """ Single Layer BN and Relui """ return nn.Sequential(mynn.Norm2d(channels), nn.ReLU(inplace=True))
def __init__(self, block, layers, wt_layer=None, num_classes=1000): self.inplanes = 64 # self.inplanes = 128 super(ResNet, self).__init__() self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) if wt_layer[2] == 1: self.bn1 = InstanceWhitening(64) self.relu = nn.ReLU(inplace=False) elif wt_layer[2] == 2: self.bn1 = InstanceWhitening(64) self.relu = nn.ReLU(inplace=False) elif wt_layer[2] == 3: self.bn1 = nn.InstanceNorm2d(64, affine=False) self.relu = nn.ReLU(inplace=True) elif wt_layer[2] == 4: self.bn1 = nn.InstanceNorm2d(64, affine=True) self.relu = nn.ReLU(inplace=True) elif wt_layer[2] == 5: self.bn1 = SyncSwitchWhiten2d(self.inplanes, num_pergroup=16, sw_type=2, T=5, tie_weight=False, eps=1e-5, momentum=0.99, affine=True) self.relu = nn.ReLU(inplace=True) else: self.bn1 = mynn.Norm2d(64) self.relu = nn.ReLU(inplace=True) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0], wt_layer=wt_layer[3]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2, wt_layer=wt_layer[4]) self.layer3 = self._make_layer(block, 256, layers[2], stride=2, wt_layer=wt_layer[5]) self.layer4 = self._make_layer(block, 512, layers[3], stride=2, wt_layer=wt_layer[6]) self.avgpool = nn.AvgPool2d(7, stride=1) self.fc = nn.Linear(512 * block.expansion, num_classes) self.wt_layer = wt_layer for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif isinstance(m, nn.BatchNorm2d) or isinstance( m, nn.SyncBatchNorm): if m.weight is not None: nn.init.constant_(m.weight, 1) if m.bias is not None: nn.init.constant_(m.bias, 0)