def __init__(self, inplanes, outplanes, expplanes, k=3, stride=1, drop_prob=0, num_steps=3e5, start_step=0, activation=nn.ReLU, act_params={"inplace": True}, SE=False): super(LinearBottleneck, self).__init__() self.conv1 = nn.Conv2d(inplanes, expplanes, kernel_size=1, bias=False) self.bn1 = nn.BatchNorm2d(expplanes) self.db1 = DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0., stop_value=drop_prob, nr_steps=num_steps, start_step=start_step) # TODO: first doesn't have act? self.conv2 = nn.Conv2d(expplanes, expplanes, kernel_size=k, stride=stride, padding=k // 2, bias=False, groups=expplanes) self.bn2 = nn.BatchNorm2d(expplanes) self.db2 = DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0., stop_value=drop_prob, nr_steps=num_steps, start_step=start_step) self.act2 = activation(**act_params) self.se = SqEx(expplanes) if SE else lambda x: x self.conv3 = nn.Conv2d(expplanes, outplanes, kernel_size=1, bias=False) self.bn3 = nn.BatchNorm2d(outplanes) self.db3 = DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0., stop_value=drop_prob, nr_steps=num_steps, start_step=start_step) self.act3 = activation(**act_params) self.stride = stride self.expplanes = expplanes self.inplanes = inplanes self.outplanes = outplanes
def __init__(self, blocks_args=None, global_params=None): super().__init__() assert isinstance(blocks_args, list), 'blocks_args should be a list' assert len(blocks_args) > 0, 'block args must be greater than 0' self._global_params = global_params self._blocks_args = blocks_args # Batch norm parameters bn_mom = 1 - self._global_params.batch_norm_momentum bn_eps = self._global_params.batch_norm_epsilon # Get stem static or dynamic convolution depending on image size image_size = global_params.image_size Conv2d = get_same_padding_conv2d(image_size=image_size) # Stem in_channels = 3 # rgb out_channels = round_filters(32, self._global_params) # number of output channels self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False) self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps) image_size = calculate_output_image_size(image_size, 2) # Build blocks self.drop_blocks = [] self.drop_blocks.append(DropBlock2D(block_size=3, drop_prob=0.3)) self.drop_blocks.append(DropBlock2D(block_size=3, drop_prob=0.3)) self._blocks = nn.ModuleList([]) for block_args in self._blocks_args: # Update block input and output filters based on depth multiplier. block_args = block_args._replace( input_filters=round_filters(block_args.input_filters, self._global_params), output_filters=round_filters(block_args.output_filters, self._global_params), num_repeat=round_repeats(block_args.num_repeat, self._global_params) ) # The first block needs to take care of stride and filter size increase. self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size)) image_size = calculate_output_image_size(image_size, block_args.stride) if block_args.num_repeat > 1: # modify block_args to keep same output size block_args = block_args._replace(input_filters=block_args.output_filters, stride=1) for _ in range(block_args.num_repeat - 1): self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size)) # image_size = calculate_output_image_size(image_size, block_args.stride) # stride = 1 # Head in_channels = block_args.output_filters # output of final block out_channels = round_filters(1280, self._global_params) Conv2d = get_same_padding_conv2d(image_size=image_size) self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False) self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps) # Final linear layer self._avg_pooling = nn.AdaptiveAvgPool2d(1) self._dropout = nn.Dropout(self._global_params.dropout_rate) self._fc = nn.Linear(out_channels, self._global_params.num_classes) self._swish = MemoryEfficientSwish()
def __init__(self, inp, oup, stride, expand_ratio, kernel, drop_prob=0.0, num_steps=3e5, activation=nn.ReLU6, act_params={"inplace": True}): super(InvertedResidual, self).__init__() self.stride = stride assert stride in [1, 2] self.use_res_connect = self.stride == 1 and inp == oup self.conv = nn.Sequential( # pw nn.Conv2d(inp, inp * expand_ratio, 1, 1, 0, bias=False), nn.BatchNorm2d(inp * expand_ratio), DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0., stop_value=drop_prob, nr_steps=num_steps), activation(**act_params), # dw nn.Conv2d(inp * expand_ratio, inp * expand_ratio, kernel, stride, kernel // 2, groups=inp * expand_ratio, bias=False), nn.BatchNorm2d(inp * expand_ratio), DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0., stop_value=drop_prob, nr_steps=num_steps), activation(**act_params), # pw-linear nn.Conv2d(inp * expand_ratio, oup, 1, 1, 0, bias=False), nn.BatchNorm2d(oup), DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0., stop_value=drop_prob, nr_steps=num_steps), ) if self.use_res_connect: self.skip_drop = DropBlockScheduled(DropBlock2D( drop_prob=drop_prob, block_size=7), start_value=0., stop_value=drop_prob, nr_steps=num_steps)
def __init__(self, senet, input3ch=True, three_neck=False, mixup_alpha=None, mix_cand_layers=None, use_mish=False, cutmix_alpha=None, cutmix_cand_layers=None, output_layers=[2, 3, 4], dropblock_p=None): super(SENetEncoder_CalibMixup_Multiscale_v2, self).__init__() self.three_neck = False self.mixup_alpha = mixup_alpha self.mix_cand_layers = mix_cand_layers self.cutmix_alpha = cutmix_alpha self.cutmix_cand_layers = cutmix_cand_layers self.output_layers = output_layers self.layer0 = senet.layer0 if not input3ch: self.layer0 = first_conv2d_3ch_to_1ch(self.layer0) self.layer1 = senet.layer1 # (256, 32, 32) self.layer2 = senet.layer2 # (512, 16, 16) self.layer3 = senet.layer3 # (1024, 8, 8) self.layer4 = senet.layer4 # (2048, 4, 4) self.calb_mixup0 = mixup.CalibrationMixup(layer_number=0) self.calb_mixup1 = mixup.CalibrationMixup(layer_number=1) self.calb_mixup2 = mixup.CalibrationMixup(layer_number=2) self.calb_mixup3 = mixup.CalibrationMixup(layer_number=3) self.dropblock0 = DropBlock2D( drop_prob=dropblock_p, block_size=10) if dropblock_p is not None else None self.dropblock1 = DropBlock2D( drop_prob=dropblock_p, block_size=5) if dropblock_p is not None else None self.dropblock2 = DropBlock2D( drop_prob=dropblock_p, block_size=5) if dropblock_p is not None else None if use_mish: mish.relu_to_mish(self.layer0) mish.relu_to_mish(self.layer1) mish.relu_to_mish(self.layer2) mish.relu_to_mish(self.layer3) mish.relu_to_mish(self.layer4) self.pooling = nn.AdaptiveAvgPool2d(1)
def __init__(self, inplanes, planes, stride=1, downsample=None, radix=2, cardinality=1, bottleneck_width=64, avd=False, avd_first=False, dilation=1, is_first=False, norm_layer=nn.BatchNorm2d, dropblock_prob=0.0, last_gamma=False): super(Bottleneck, self).__init__() group_width = int(planes * (bottleneck_width / 64.)) * cardinality self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False) self.bn1 = norm_layer(group_width) self.dropblock_prob = dropblock_prob self.radix = radix self.avd = avd and (stride > 1 or is_first) self.avd_first = avd_first if self.avd: self.avd_layer = nn.AvgPool2d(3, stride, padding=1) stride = 1 if dropblock_prob > 0.0: self.dropblock1 = DropBlock2D(dropblock_prob, 3) if radix == 1: self.dropblock2 = DropBlock2D(dropblock_prob, 3) self.dropblock3 = DropBlock2D(dropblock_prob, 3) if radix >= 1: self.conv2 = SplAtConv2d( group_width, group_width, kernel_size=3, stride=stride, padding=dilation, dilation=dilation, groups=cardinality, bias=False, radix=radix, norm_layer=norm_layer, dropblock_prob=dropblock_prob) else: self.conv2 = nn.Conv2d( group_width, group_width, kernel_size=3, stride=stride, padding=dilation, dilation=dilation, groups=cardinality, bias=False) self.bn2 = norm_layer(group_width) self.conv3 = nn.Conv2d( group_width, planes * 4, kernel_size=1, bias=False) self.bn3 = norm_layer(planes*4) if last_gamma: from torch.nn.init import zeros_ zeros_(self.bn3.weight) self.relu = nn.ReLU(inplace=True) self.downsample = downsample self.dilation = dilation self.stride = stride
def __init__(self, in_channel, D_filters, drop_prob=0.1, use_block=False, block_size=5): super().__init__() self.D = D_filters self.conv1 = nn.Conv2d(in_channel, self.D, 3, padding=1) self.bn1 = nn.BatchNorm2d(self.D) self.conv2 = nn.Conv2d(self.D, self.D, 3, padding=1) self.bn2 = nn.BatchNorm2d(self.D) self.conv3 = nn.Conv2d(self.D, self.D, 3, padding=1) self.bn3 = nn.BatchNorm2d(self.D) self.jump_conv = nn.Conv2d(in_channel, self.D, 1) self.maxpool = nn.MaxPool2d(2, 2) if use_block: self.drop = DropBlock2D(block_size=block_size, drop_prob=drop_prob) else: self.drop = nn.Dropout(p=drop_prob) nn.init.kaiming_normal_(self.conv1.weight) nn.init.kaiming_normal_(self.conv2.weight) nn.init.kaiming_normal_(self.conv3.weight) nn.init.kaiming_normal_(self.jump_conv.weight)
def conv_bn(inp, out): return nn.Sequential( nn.Conv2d(inp, out, kernel_size=3, stride=1, padding=1, bias=False), nn.BatchNorm2d(out), DropBlock2D(block_size=3, drop_prob=0.1), nn.PReLU(out) )
def __init__(self, in_ch, out_ch, circular_padding, bilinear=True, group_conv=False, use_dropblock=False, drop_p=0.5): super(up, self).__init__() # would be a nice idea if the upsampling could be learned too, # but my machine do not have enough memory to handle all those weights if bilinear: self.up = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True) elif group_conv: self.up = nn.ConvTranspose2d(in_ch // 2, in_ch // 2, 2, stride=2, groups=in_ch // 2) else: self.up = nn.ConvTranspose2d(in_ch // 2, in_ch // 2, 2, stride=2) if circular_padding: self.conv = double_conv_circular(in_ch, out_ch, group_conv=group_conv) else: self.conv = double_conv(in_ch, out_ch, group_conv=group_conv) self.use_dropblock = use_dropblock if self.use_dropblock: self.dropblock = DropBlock2D(block_size=7, drop_prob=drop_p)
def __init__(self, vgg_name='VGG19', in_channel=3, out_channel=10, drop_prob=0.0, block_size=5, forward='dropblock'): super(VGG2, self).__init__() # 修改了 make_layer 这个部分, 源码是在上面, 把中间分层处理 self.f1, self.f2, self.f3 = self._make_layers2(vggcfg[vgg_name], in_channel=in_channel) self.classifier = nn.Linear(512, out_channel) self.dropblock = LinearScheduler(DropBlock2D(drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=5e3) self.wh = 2 self.wh2 = 4 self.align_sche = False self.i = 0 # self.cr = CropAndResize(8, 8) # 8 is according to the real size self.cr = RoIAlign(self.wh, self.wh, transform_fpcoor=True) self.cr2 = RoIAlign(self.wh2, self.wh2, transform_fpcoor=True) # 注释掉 其中的一个, if forward == 'dropblock': self.forward = self._forward_dropblock print("------- VGG with Dropblock ---------\n") else: self.forward = self._forward_align print("------- VGG with ROiAlign ---------\n")
def __init__(self, feature_dim, with_norm='none', upsample_method='bilinear'): super(FPT, self).__init__() self.feature_dim = feature_dim assert upsample_method in ['nearest', 'bilinear'] def interpolate(input): return F.interpolate(input, scale_factor=2, mode=upsample_method, align_corners=False if upsample_method == 'bilinear' else None) self.fpn_upsample = interpolate assert with_norm in ['group_norm', 'batch_norm', 'none'] if with_norm == 'batch_norm': norm = nn.BatchNorm2d elif with_norm == 'group_norm': def group_norm(num_channels): return nn.GroupNorm(32, num_channels) norm = group_norm self.st_p5 = SelfTrans(n_head = 1, n_mix = 2, d_model = feature_dim, d_k= feature_dim, d_v= feature_dim) self.st_p4 = SelfTrans(n_head = 1, n_mix = 2, d_model = feature_dim, d_k= feature_dim, d_v= feature_dim) self.st_p3 = SelfTrans(n_head = 1, n_mix = 2, d_model = feature_dim, d_k= feature_dim, d_v= feature_dim) self.st_p2 = SelfTrans(n_head = 1, n_mix = 2, d_model = feature_dim, d_k= feature_dim, d_v= feature_dim) self.gt_p4_p5 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True) self.gt_p3_p4 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True) self.gt_p3_p5 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True) self.gt_p2_p3 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True) self.gt_p2_p4 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True) self.gt_p2_p5 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True) self.rt_p5_p4 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False) self.rt_p5_p3 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False) self.rt_p5_p2 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False) self.rt_p4_p3 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False) self.rt_p4_p2 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False) self.rt_p3_p2 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False) drop_block = DropBlock2D(block_size=3, drop_prob=0.2) if with_norm != 'none': self.fpn_p5_1x1 = nn.Sequential(*[nn.Conv2d(2048, feature_dim, 1, bias=False), norm(feature_dim)]) self.fpn_p4_1x1 = nn.Sequential(*[nn.Conv2d(1024, feature_dim, 1, bias=False), norm(feature_dim)]) self.fpn_p3_1x1 = nn.Sequential(*[nn.Conv2d(512, feature_dim, 1, bias=False), norm(feature_dim)]) self.fpn_p2_1x1 = nn.Sequential(*[nn.Conv2d(256, feature_dim, 1, bias=False), norm(feature_dim)]) self.fpt_p5 = nn.Sequential(*[nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1, bias=False), norm(feature_dim)]) self.fpt_p4 = nn.Sequential(*[nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1, bias=False), norm(feature_dim)]) self.fpt_p3 = nn.Sequential(*[nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1, bias=False), norm(feature_dim)]) self.fpt_p2 = nn.Sequential(*[nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1, bias=False), norm(feature_dim)]) else: self.fpn_p5_1x1 = nn.Conv2d(2048, feature_dim, 1) self.fpn_p4_1x1 = nn.Conv2d(1024, feature_dim, 1) self.fpn_p3_1x1 = nn.Conv2d(512, feature_dim, 1) self.fpn_p2_1x1 = nn.Conv2d(256, feature_dim, 1) self.fpt_p5 = nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1) self.fpt_p4 = nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1) self.fpt_p3 = nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1) self.fpt_p2 = nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1) self.initialize()
def __init__(self, cfg, block, layers): self.inplanes = 64 super(ResNet, self).__init__() self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) #150*150 self.bn1 = nn.BatchNorm2d(64) self.relu = nn.ReLU(inplace=True) self.cfg = cfg #----------new structure called DropBlock 11sr April------------------------- if self.cfg.MODEL.BACKBONE.DROP_BLOCK: drop_prob = 0.5 block_size = 3 self.dropblock = LinearScheduler(DropBlock2D( drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=5) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) #75*75 self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2) #38*38 self.layer3 = self._make_layer(block, 256, layers[2], stride=2) #19*19 #self.layer4 = self._make_layer(block, 512,layers[3] , stride=1) #10*10 self.ex_layer0 = self._make_layer(block, 512, 2, stride=2) #10*10 #self.maxpoo2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) #2. extra_layers (ReLU will be used in the foward function) 10thApril,Xiaoyu Zhu # if cfg.MODEL.BACKBONE.DEPTH>34: # self.ex_layer1 = nn.Sequential(BasicBlock_modified(2048,512)) # #self.ex_layer1 = self._make_extra_layers(2048,512,3,1) #5*5 # else: # self.ex_layer1 = nn.Sequential(BasicBlock_modified(512,512)) self.ex_layer1 = self._make_layer(block, 512, 1, stride=2) #5*5 #self.maxpoo3 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.ex_layer2 = self._make_layer(block, 256, 1, stride=2) #3*3 #self.maxpoo4 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.ex_layer3 = self._make_layer(block, 128, 1, stride=2) # if cfg.MODEL.BACKBONE.DEPTH>34: # self.ex_layer3 = self._make_extra_layers(256*4,128,[2,3],0) # else: # #self.ex_layer3 = self._make_extra_layers(256,128,[2,3],0) # self.ex_layer3 = self._make_layer(block, 128, 1, stride=2) #BasicBlock_modified(inplanes=256, planes=128, kernel=[2,3],stride=2, padding=0) # kaiming weight normal after default initialization for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_()
def test_block_mask_hW_odd(): db = DropBlock2D(block_size=3, drop_prob=0.1) mask = torch.tensor([[[0., 0., 0., 1., 0.], [0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.]]]) expected = torch.tensor([[[1., 1., 0., 0., 0.], [1., 1., 0., 0., 0.], [1., 1., 1., 1., 1.], [1., 1., 1., 1., 1.]]]) block_mask = db._compute_block_mask(mask) assert torch.equal(block_mask, expected)
def test_forward_pass2(): block_sizes = [2, 3, 4, 5, 6, 7, 8] heights = [5, 6, 8, 10, 11, 14, 15] widths = [5, 7, 8, 10, 15, 14, 15] for block_size, height, width in zip(block_sizes, heights, widths): dropout = DropBlock2D(0.1, block_size=block_size) input = torch.randn((5, 20, height, width)) output = dropout(input) assert tuple(input.shape) == tuple(output.shape)
def __init__(self, filters_in): super(MSR_Convset_S, self).__init__() self.__dw0 = Convolutional(filters_in=filters_in, filters_out=filters_in*2, kernel_size=3, stride=1, pad=1, norm="bn", activate="leaky") self.__pw0 = Convolutional(filters_in=filters_in*2, filters_out=filters_in, kernel_size=1, stride=1, pad=0, norm="bn", activate="leaky") self.__dw1 = Convolutional(filters_in=filters_in, filters_out=filters_in*2, kernel_size=3, stride=1, pad=1, dila=1, norm="bn", activate="leaky") self.__pw1 = Convolutional(filters_in=filters_in*2, filters_out=filters_in, kernel_size=1, stride=1, pad=0, norm="bn", activate="leaky") self.__drop = LinearScheduler(DropBlock2D(block_size=3, drop_prob=0.1), start_value=0., stop_value=0.1, nr_steps=5)
def __init__(self, in_channels, channels, kernel_size, stride=(1, 1), padding=(0, 0), dilation=(1, 1), groups=1, bias=True, radix=2, reduction_factor=4, rectify=False, rectify_avg=False, norm_layer=None, dropblock_prob=0.0, **kwargs): super(SplAtConv2d, self).__init__() padding = _pair(padding) self.rectify = rectify and (padding[0] > 0 or padding[1] > 0) self.rectify_avg = rectify_avg inter_channels = max(in_channels * radix // reduction_factor, 32) self.radix = radix self.cardinality = groups self.channels = channels self.dropblock_prob = dropblock_prob self.conv = nn.Conv2d(in_channels, channels * radix, kernel_size, stride, padding, dilation, groups=groups * radix, bias=bias, **kwargs) self.use_bn = norm_layer is not None if self.use_bn: self.bn0 = norm_layer(channels * radix) self.relu = nn.ReLU(inplace=True) self.fc1 = nn.Conv2d(channels, inter_channels, 1, groups=self.cardinality) if self.use_bn: self.bn1 = norm_layer(inter_channels) self.fc2 = nn.Conv2d(inter_channels, channels * radix, 1, groups=self.cardinality) if dropblock_prob > 0.0: self.dropblock = DropBlock2D(dropblock_prob, 3) self.rsoftmax = rSoftMax(radix, groups)
def __init__(self, block, layers, num_classes=1000, drop_prob=0., block_size=5): super(ResNet, self).__init__() self.inplanes = 64 self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) self.bn1 = nn.BatchNorm2d(64) self.relu = nn.ReLU(inplace=True) self.dropblock = LinearScheduler(DropBlock2D(drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=5e3) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2) self.layer3 = self._make_layer(block, 256, layers[2], stride=2) self.layer4 = self._make_layer(block, 512, layers[3], stride=2) self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) self.fc = nn.Linear(512 * block.expansion, num_classes) self.wh = 2 self.wh2 = 8 self.align_sche = False self.i = 0 # self.cr = CropAndResize(8, 8) # 8 is according to the real size self.cr = RoIAlign(self.wh, self.wh, transform_fpcoor=True) self.cr2 = RoIAlign(self.wh2, self.wh2, transform_fpcoor=True) print("--------------------------------------------------------" "\n-------- RoiAlign -------\n" "--------------------------------------------------------") for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif isinstance(m, nn.BatchNorm2d): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0)
def __init__(self, block, num_blocks, num_classes=10, drop_prob=0., block_size=5): super(ResNet, self).__init__() self.in_planes = 64 self.conv1 = conv3x3(3, 64) self.bn1 = nn.BatchNorm2d(64) self.dropblock = LinearScheduler( DropBlock2D(drop_prob=drop_prob, block_size=block_size, att=True), start_value=0., stop_value=drop_prob, nr_steps=5e4 ) self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1) self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2) self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2) self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2) self.linear = nn.Linear(512 * block.expansion, num_classes)
def __init__(self, block, layers, num_classes=1000, drop_prob=0., block_size=5, gkernel=False): super(ResNet, self).__init__() self.inplanes = 64 self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) self.bn1 = nn.BatchNorm2d(64) self.relu = nn.ReLU(inplace=True) self.dropblock = LinearScheduler(DropBlock2D(drop_prob=drop_prob, block_size=block_size, gkernel=gkernel), start_value=0., stop_value=drop_prob, nr_steps=5e3) self.dropcblock = LinearScheduler(DropCBlock(drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=5e3) # self.dropchannel = DropChannel(drop_prob=drop_prob) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2) self.layer3 = self._make_layer(block, 256, layers[2], stride=2) self.layer4 = self._make_layer(block, 512, layers[3], stride=2) self.avgpool = nn.AdaptiveAvgPool2d((1, 1)) self.fc = nn.Linear(512 * block.expansion, num_classes) for m in self.modules(): if isinstance(m, nn.Conv2d): nn.init.kaiming_normal_(m.weight, mode='fan_out', nonlinearity='relu') elif isinstance(m, nn.BatchNorm2d): nn.init.constant_(m.weight, 1) nn.init.constant_(m.bias, 0)
def __init__(self, input_channels, input_width, num_classes=-1, dropout_prob=0, drop_prob=0, block_size=5): super(SimpleCNN, self).__init__() if drop_prob != 0: self.dropblock = LinearScheduler( DropBlock2D(drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=5e3 ) self.drop_prob = drop_prob self.block_size = block_size self.dropout_prob = dropout_prob self.dim = input_width self.conv1 = nn.Conv2d(input_channels, 50, kernel_size=3) self.conv2 = nn.Conv2d(50, 100, kernel_size=5) self.conv3 = nn.Conv2d(100, 200, kernel_size=5) self.conv4 = nn.Conv2d(200, 400, kernel_size=2) self.fc1 = nn.Linear(400, 300) self.fc2 = nn.Linear(300, num_classes)
def __init__(self, in_planes, out_planes, stride, dropRate=0.0, my_conv=Conv2d): super(BasicBlock, self).__init__() self.bn1 = nn.BatchNorm2d(in_planes) # self.bn1 = nn.GroupNorm(num_groups=8,num_channels=in_planes) # self.relu1 = nn.ReLU(inplace=True) self.relu1 = Mish() self.conv1 = my_conv(in_planes, out_planes, kernel_size=3, stride=stride, padding=1, bias=False) self.bn2 = nn.BatchNorm2d(out_planes) # self.bn2 = nn.GroupNorm(num_groups=8,num_channels=out_planes) # self.relu2 = nn.ReLU(inplace=True) self.relu2 = Mish() self.conv2 = my_conv(out_planes, out_planes, kernel_size=3, stride=1, padding=1, bias=False) self.droprate = dropRate from dropblock import LinearScheduler, DropBlock2D self.conv2_drop = LinearScheduler(DropBlock2D(drop_prob=dropRate, block_size=3), start_value=0, stop_value=dropRate, nr_steps=10000) self.equalInOut = (in_planes == out_planes) self.convShortcut = (not self.equalInOut) and my_conv( in_planes, out_planes, kernel_size=1, stride=stride, padding=0, bias=False) or None
def test_forward_pass(): db = DropBlock2D(block_size=3, drop_prob=0.1) block_mask = torch.tensor([[[0., 0., 0., 1., 1., 1., 1.], [0., 0., 0., 0., 0., 0., 1.], [0., 0., 0., 0., 0., 0., 1.], [1., 1., 1., 0., 0., 0., 1.], [1., 1., 1., 1., 1., 1., 1.], [1., 1., 1., 1., 1., 1., 1.], [1., 1., 1., 1., 1., 1., 1.]]]) db._compute_block_mask = mock.MagicMock(return_value=block_mask) x = torch.ones(10, 10, 7, 7) h = db(x) expected = block_mask * block_mask.numel() / block_mask.sum() expected = expected[:, None, :, :].expand_as(x) assert tuple(h.shape) == (10, 10, 7, 7) assert torch.equal(h, expected)
def __init__(self, cfg, block, layers): self.inplanes = 64 super(ResNet, self).__init__() self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) #150*150 self.bn1 = nn.BatchNorm2d(64) self.relu = nn.ReLU(inplace=True) self.cfg = cfg #----------new structure called DropBlock 11sr April------------------------- if self.cfg.MODEL.BACKBONE.DROP_BLOCK: drop_prob = 0.5 block_size = 3 self.dropblock = LinearScheduler(DropBlock2D( drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=5) self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2) #75*75 self.layer3 = self._make_layer(block, 256, layers[2], stride=2) #38*38 #2. extra_layers (ReLU will be used in the foward function) 10thApril,Xiaoyu Zhu self.ex_layer00 = self._make_layer(block, 512, 1, stride=2) #19*19 self.ex_layer0 = self._make_layer(block, 256, 1, stride=2) #10*10 self.ex_layer1 = self._make_layer(block, 256, 1, stride=2) #5*5 self.ex_layer2 = self._make_layer(block, 128, 1, stride=2) #3*3 self.ex_layer3 = self._make_layer(block, 128, 1, stride=2) #1*2 # kaiming weight normal after default initialization for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_()
def __init__(self, cfg, block, layers): self.inplanes = 64 super(ResNet, self).__init__() self.conv1 = nn.Conv2d(3, 64, kernel_size=7, stride=2, padding=3, bias=False) #150*150 self.bn1 = nn.BatchNorm2d(64) self.relu = nn.ReLU(inplace=True) self.cfg = cfg #----------new structure called DropBlock 11sr April------------------------- if self.cfg.MODEL.BACKBONE.DROP_BLOCK: drop_prob = 0. block_size = 5 self.dropblock = LinearScheduler(DropBlock2D( drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=5) self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) #75*75 self.layer1 = self._make_layer(block, 64, layers[0]) self.layer2 = self._make_layer(block, 128, layers[1], stride=2) #38*38 self.layer3 = self._make_layer(block, 256, layers[2], stride=2) #19*19 self.layer4 = self._make_layer(block, 512, layers[3], stride=2) #10*10 #2. extra_layers (ReLU will be used in the foward function) 10thApril,Xiaoyu Zhu if cfg.MODEL.BACKBONE.DEPTH > 34: self.ex_layer1 = self._make_extra_layers(2048, 512, 3, 1) #5*5 else: self.ex_layer1 = self._make_extra_layers(512, 512, 3, 1) #5*5 self.ex_layer2 = self._make_extra_layers(512, 256, 3, 1) #3*3 self.ex_layer3 = self._make_extra_layers(256, 128, [2, 3], 0) #1*1 #-----------------------------------Old Version------------------- # 2. extra_layers (ReLU will be used in the foward function) # extra_layer=[] #This is the old_version of the arch. But I want to add some batch_norm in to the layers # if cfg.MODEL.BACKBONE.DEPTH>34: # extra_layer.append(torch.nn.Conv2d(in_channels=2048, out_channels=256,kernel_size=3,stride=1,padding=1)) # else: # extra_layer.append(torch.nn.Conv2d(in_channels=512, out_channels=256,kernel_size=3,stride=1,padding=1)) # #need ReLU # extra_layer.append(torch.nn.Conv2d(in_channels=256, out_channels=512,kernel_size=3,stride=2,padding=1)) #5*5 # #need ReLU # extra_layer.append(torch.nn.Conv2d(in_channels=512, out_channels=512,kernel_size=3,stride=1,padding=1)) # #need ReLU # extra_layer.append(torch.nn.Conv2d(in_channels=512, out_channels=256,kernel_size=3,stride=2,padding=1)) #3*3 # #need ReLU # extra_layer.append(torch.nn.Conv2d(in_channels=256, out_channels=256,kernel_size=3,stride=1,padding=1)) # #need ReLU # extra_layer.append(torch.nn.Conv2d(in_channels=256, out_channels=128,kernel_size=3,stride=2,padding=0)) #1*1 # #need ReLU # self.extra_layer=torch.nn.Sequential(*extra_layer) #----------------------------------------------------------------- # kaiming weight normal after default initialization for m in self.modules(): if isinstance(m, nn.Conv2d): n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels m.weight.data.normal_(0, math.sqrt(2. / n)) elif isinstance(m, nn.BatchNorm2d): m.weight.data.fill_(1) m.bias.data.zero_()
def __init__(self, senet, input3ch=True, three_neck=False, mixup_alpha=None, mix_cand_layers=None, use_mish=False, cutmix_alpha=None, cutmix_cand_layers=None, output_layer3=False, output_layer2=False, dropblock_p=None, upsample_size=None, calib_mixup=False): super(SENetEncoder_Mixup, self).__init__() self.three_neck = three_neck self.mixup_alpha = mixup_alpha self.mix_cand_layers = mix_cand_layers self.cutmix_alpha = cutmix_alpha self.cutmix_cand_layers = cutmix_cand_layers self.output_layer3 = output_layer3 self.output_layer2 = output_layer2 self.layer0 = senet.layer0 if not input3ch: self.layer0 = first_conv2d_3ch_to_1ch(self.layer0) self.layer1 = senet.layer1 self.layer2 = senet.layer2 self.layer3 = senet.layer3 if three_neck: self.layer4_1 = copy_model(senet.layer4) self.layer4_2 = copy_model(senet.layer4) self.layer4_3 = copy_model(senet.layer4) else: self.layer4 = copy_model(senet.layer4) self.calb_mixup0 = mixup.CalibrationMixup( layer_number=0) if calib_mixup else None self.calb_mixup1 = mixup.CalibrationMixup( layer_number=1) if calib_mixup else None self.calb_mixup2 = mixup.CalibrationMixup( layer_number=2) if calib_mixup else None self.calb_mixup3 = mixup.CalibrationMixup( layer_number=3) if calib_mixup else None self.dropblock0 = DropBlock2D( drop_prob=dropblock_p, block_size=10) if dropblock_p is not None else None self.dropblock1 = DropBlock2D( drop_prob=dropblock_p, block_size=5) if dropblock_p is not None else None self.dropblock2 = DropBlock2D( drop_prob=dropblock_p, block_size=5) if dropblock_p is not None else None self.upsample = nn.Upsample( size=upsample_size, mode='bilinear', align_corners=True) if upsample_size is not None else None if use_mish: mish.relu_to_mish(self.layer0) mish.relu_to_mish(self.layer1) mish.relu_to_mish(self.layer2) mish.relu_to_mish(self.layer3) if three_neck: mish.relu_to_mish(self.layer4_1) mish.relu_to_mish(self.layer4_2) mish.relu_to_mish(self.layer4_3) else: mish.relu_to_mish(self.layer4)
def test_large_block_size(): dropout = DropBlock2D(0.3, block_size=9) x = torch.rand(100, 10, 16, 16) output = dropout(x) assert tuple(x.shape) == tuple(output.shape)
def __init__(self, num_classes=1000, drop_prob=0.3, block_size=3): """ Constructor Args: num_classes: number of classes """ super(Xception_drop, self).__init__() self.num_classes = num_classes self.conv1 = nn.Conv2d(3, 32, 3, 2, 0, bias=False) self.bn1 = nn.BatchNorm2d(32) self.relu1 = nn.ReLU(inplace=True) self.conv2 = nn.Conv2d(32, 64, 3, bias=False) self.bn2 = nn.BatchNorm2d(64) self.relu2 = nn.ReLU(inplace=True) #do relu here self.dropblock = LinearScheduler(DropBlock2D(drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=int(5e3)) self.block1 = Block(64, 128, 2, 2, start_with_relu=False, grow_first=True) self.block2 = Block(128, 256, 2, 2, start_with_relu=True, grow_first=True) self.block3 = Block(256, 728, 2, 2, start_with_relu=True, grow_first=True) self.block4 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) self.block5 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) self.block6 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) self.block7 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) self.block8 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) self.block9 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) self.block10 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) self.block11 = Block(728, 728, 3, 1, start_with_relu=True, grow_first=True) self.block12 = Block(728, 1024, 2, 2, start_with_relu=True, grow_first=False) self.conv3 = SeparableConv2d(1024, 1536, 3, 1, 1) self.bn3 = nn.BatchNorm2d(1536) self.relu3 = nn.ReLU(inplace=True) #do relu here self.conv4 = SeparableConv2d(1536, 2048, 3, 1, 1) self.bn4 = nn.BatchNorm2d(2048) self.fc = nn.Linear(2048, num_classes)
def test_forward_pass_with_cuda(): dropout = DropBlock2D(0.3, block_size=5).to('cuda') x = torch.rand(100, 10, 16, 16).to('cuda') output = dropout(x) assert tuple(x.shape) == tuple(output.shape)
def __init__(self, output_class=7, model_path=None, resnet_type=34, drop_block=False, drop_prob=0.5, drop_pos=None, layer_depth=4, drop_block_size=7): super(ResNetDropblock, self).__init__() assert resnet_type in [18, 34, 50] assert layer_depth in [1, 2, 3, 4] if resnet_type == 18: self.base = resnet18(pretrained=False, num_classes=1000) # state_dict = torch.load('resnet18.pth') if layer_depth == 4: last_fc_in_channel = 512 * 1 elif layer_depth == 3: last_fc_in_channel = 256 * 1 elif layer_depth == 2: last_fc_in_channel = 128 * 1 else: # elif layer_depth == 1: last_fc_in_channel = 64 * 1 elif resnet_type == 34: self.base = resnet34(pretrained=False, num_classes=1000) # state_dict = torch.load('resnet34.pth') if layer_depth == 4: last_fc_in_channel = 512 * 1 elif layer_depth == 3: last_fc_in_channel = 256 * 1 elif layer_depth == 2: last_fc_in_channel = 128 * 1 else: # elif layer_depth == 1: last_fc_in_channel = 64 * 1 else: # elif resnet_type == 50: self.base = resnet50(pretrained=False, num_classes=1000) # state_dict = torch.load('resnet50.pth') if layer_depth == 4: last_fc_in_channel = 512 * 4 elif layer_depth == 3: last_fc_in_channel = 256 * 4 elif layer_depth == 2: last_fc_in_channel = 128 * 4 else: # elif layer_depth == 1: last_fc_in_channel = 64 * 4 # self.base.load_state_dict(state_dict) # def weight_init(m): # if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d): # nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('relu')) # if m.bias is not None: # nn.init.zeros_(m.bias) # # self.base.layer3.apply(weight_init) # self.base.layer4.apply(weight_init) self.base.fc = nn.Linear(in_features=last_fc_in_channel, out_features=output_class, bias=True) if drop_block: self.dropblock = LinearScheduler( DropBlock2D(drop_prob=drop_prob, block_size=drop_block_size), start_value=0., stop_value=drop_prob, nr_steps=300 ) else: self.dropblock = nn.Sequential() self.drop_pos = drop_pos self.layer_depth = layer_depth if model_path is not None: self.model_path = model_path assert '.pth' in self.model_path or '.pkl' in self.model_path self.init_weight()
def __init__( self, block, layers, groups, reduction, dropout_p=0.2, inplanes=128, input_3x3=True, downsample_kernel_size=3, downsample_padding=1, num_classes=1000, strides=[ 2, ] * 5, adaptive_pool=False, input_c=3, ): """ Parameters ---------- block (nn.Module): Bottleneck class. - For SENet154: SEBottleneck - For SE-ResNet models: SEResNetBottleneck - For SE-ResNeXt models: SEResNeXtBottleneck layers (list of ints): Number of residual blocks for 4 layers of the network (layer1...layer4). groups (int): Number of groups for the 3x3 convolution in each bottleneck block. - For SENet154: 64 - For SE-ResNet models: 1 - For SE-ResNeXt models: 32 reduction (int): Reduction ratio for Squeeze-and-Excitation modules. - For all models: 16 dropout_p (float or None): Drop probability for the Dropout layer. If `None` the Dropout layer is not used. - For SENet154: 0.2 - For SE-ResNet models: None - For SE-ResNeXt models: None inplanes (int): Number of input channels for layer1. - For SENet154: 128 - For SE-ResNet models: 64 - For SE-ResNeXt models: 64 input_3x3 (bool): If `True`, use three 3x3 convolutions instead of a single 7x7 convolution in layer0. - For SENet154: True - For SE-ResNet models: False - For SE-ResNeXt models: False downsample_kernel_size (int): Kernel size for downsampling convolutions in layer2, layer3 and layer4. - For SENet154: 3 - For SE-ResNet models: 1 - For SE-ResNeXt models: 1 downsample_padding (int): Padding for downsampling convolutions in layer2, layer3 and layer4. - For SENet154: 1 - For SE-ResNet models: 0 - For SE-ResNeXt models: 0 num_classes (int): Number of outputs in `last_linear` layer. - For all models: 1000 """ super(SENet, self).__init__() self.inplanes = inplanes if input_3x3: layer0_modules = [ ('conv1', nn.Conv2d(input_c, 64, 3, stride=strides[0], padding=1, bias=False)), ('bn1', nn.BatchNorm2d(64)), ('relu1', nn.ReLU(inplace=True)), ('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1, bias=False)), ('bn2', nn.BatchNorm2d(64)), ('relu2', nn.ReLU(inplace=True)), ('conv3', nn.Conv2d(64, inplanes, 3, stride=1, padding=1, bias=False)), ('bn3', nn.BatchNorm2d(inplanes)), ('relu3', nn.ReLU(inplace=True)), ] else: layer0_modules = [ ('conv1', nn.Conv2d(input_c, inplanes, kernel_size=7, stride=strides[0], padding=3, bias=False)), ('bn1', nn.BatchNorm2d(inplanes)), ('relu1', nn.ReLU(inplace=True)), ] # To preserve compatibility with Caffe weights `ceil_mode=True` # is used instead of `padding=1`. layer0_modules.append( ('pool', nn.MaxPool2d(3, stride=strides[1], ceil_mode=True))) self.layer0 = nn.Sequential(OrderedDict(layer0_modules)) self.layer1 = self._make_layer(block, planes=64, blocks=layers[0], groups=groups, reduction=reduction, downsample_kernel_size=1, downsample_padding=0) self.layer2 = self._make_layer( block, planes=128, blocks=layers[1], stride=strides[2], groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding) self.layer3 = self._make_layer( block, planes=256, blocks=layers[2], stride=strides[3], groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding) self.layer4 = self._make_layer( block, planes=512, blocks=layers[3], stride=strides[4], groups=groups, reduction=reduction, downsample_kernel_size=downsample_kernel_size, downsample_padding=downsample_padding) self.dropblock0 = DropBlock2D(drop_prob=0.2, block_size=16) self.dropblock1 = DropBlock2D(drop_prob=0.2, block_size=8) if adaptive_pool: self.avg_pool = nn.AdaptiveAvgPool2d(output_size=1) else: self.avg_pool = nn.AvgPool2d(7, stride=1) self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None self.last_linear = nn.Linear(512 * block.expansion, num_classes)
def __init__(self, num_class=11): super(Resnet50Mod, self).__init__() self.cnn = nn.Sequential(*list(origin_net.children())[:-1]) self.hidden_layer = nn.Linear(2048, 128) self.dropout = DropBlock2D(block_size=3, drop_prob=0.2) self.output = nn.Linear(128, num_class) # 11 or 10, 1 * 11