def __init__(self, inplanes, outplanes, expplanes, k=3, stride=1, drop_prob=0, num_steps=3e5, start_step=0,
activation=nn.ReLU, act_params={"inplace": True}, SE=False):
super(LinearBottleneck, self).__init__()
self.conv1 = nn.Conv2d(inplanes, expplanes, kernel_size=1, bias=False)
self.bn1 = nn.BatchNorm2d(expplanes)
self.db1 = DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0.,
stop_value=drop_prob, nr_steps=num_steps, start_step=start_step)
# TODO: first doesn't have act?
self.conv2 = nn.Conv2d(expplanes, expplanes, kernel_size=k, stride=stride, padding=k // 2, bias=False,
groups=expplanes)
self.bn2 = nn.BatchNorm2d(expplanes)
self.db2 = DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0.,
stop_value=drop_prob, nr_steps=num_steps, start_step=start_step)
self.act2 = activation(**act_params)
self.se = SqEx(expplanes) if SE else lambda x: x
self.conv3 = nn.Conv2d(expplanes, outplanes, kernel_size=1, bias=False)
self.bn3 = nn.BatchNorm2d(outplanes)
self.db3 = DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7), start_value=0.,
stop_value=drop_prob, nr_steps=num_steps, start_step=start_step)
self.act3 = activation(**act_params)
self.stride = stride
self.expplanes = expplanes
self.inplanes = inplanes
self.outplanes = outplanes
def __init__(self, blocks_args=None, global_params=None):
super().__init__()
assert isinstance(blocks_args, list), 'blocks_args should be a list'
assert len(blocks_args) > 0, 'block args must be greater than 0'
self._global_params = global_params
self._blocks_args = blocks_args
# Batch norm parameters
bn_mom = 1 - self._global_params.batch_norm_momentum
bn_eps = self._global_params.batch_norm_epsilon
# Get stem static or dynamic convolution depending on image size
image_size = global_params.image_size
Conv2d = get_same_padding_conv2d(image_size=image_size)
# Stem
in_channels = 3 # rgb
out_channels = round_filters(32, self._global_params) # number of output channels
self._conv_stem = Conv2d(in_channels, out_channels, kernel_size=3, stride=2, bias=False)
self._bn0 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
image_size = calculate_output_image_size(image_size, 2)
# Build blocks
self.drop_blocks = []
self.drop_blocks.append(DropBlock2D(block_size=3, drop_prob=0.3))
self.drop_blocks.append(DropBlock2D(block_size=3, drop_prob=0.3))
self._blocks = nn.ModuleList([])
for block_args in self._blocks_args:
# Update block input and output filters based on depth multiplier.
block_args = block_args._replace(
input_filters=round_filters(block_args.input_filters, self._global_params),
output_filters=round_filters(block_args.output_filters, self._global_params),
num_repeat=round_repeats(block_args.num_repeat, self._global_params)
)
# The first block needs to take care of stride and filter size increase.
self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))
image_size = calculate_output_image_size(image_size, block_args.stride)
if block_args.num_repeat > 1: # modify block_args to keep same output size
block_args = block_args._replace(input_filters=block_args.output_filters, stride=1)
for _ in range(block_args.num_repeat - 1):
self._blocks.append(MBConvBlock(block_args, self._global_params, image_size=image_size))
# image_size = calculate_output_image_size(image_size, block_args.stride) # stride = 1
# Head
in_channels = block_args.output_filters # output of final block
out_channels = round_filters(1280, self._global_params)
Conv2d = get_same_padding_conv2d(image_size=image_size)
self._conv_head = Conv2d(in_channels, out_channels, kernel_size=1, bias=False)
self._bn1 = nn.BatchNorm2d(num_features=out_channels, momentum=bn_mom, eps=bn_eps)
# Final linear layer
self._avg_pooling = nn.AdaptiveAvgPool2d(1)
self._dropout = nn.Dropout(self._global_params.dropout_rate)
self._fc = nn.Linear(out_channels, self._global_params.num_classes)
self._swish = MemoryEfficientSwish()
def __init__(self,
inp,
oup,
stride,
expand_ratio,
kernel,
drop_prob=0.0,
num_steps=3e5,
activation=nn.ReLU6,
act_params={"inplace": True}):
super(InvertedResidual, self).__init__()
self.stride = stride
assert stride in [1, 2]
self.use_res_connect = self.stride == 1 and inp == oup
self.conv = nn.Sequential(
# pw
nn.Conv2d(inp, inp * expand_ratio, 1, 1, 0, bias=False),
nn.BatchNorm2d(inp * expand_ratio),
DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7),
start_value=0.,
stop_value=drop_prob,
nr_steps=num_steps),
activation(**act_params),
# dw
nn.Conv2d(inp * expand_ratio,
inp * expand_ratio,
kernel,
stride,
kernel // 2,
groups=inp * expand_ratio,
bias=False),
nn.BatchNorm2d(inp * expand_ratio),
DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7),
start_value=0.,
stop_value=drop_prob,
nr_steps=num_steps),
activation(**act_params),
# pw-linear
nn.Conv2d(inp * expand_ratio, oup, 1, 1, 0, bias=False),
nn.BatchNorm2d(oup),
DropBlockScheduled(DropBlock2D(drop_prob=drop_prob, block_size=7),
start_value=0.,
stop_value=drop_prob,
nr_steps=num_steps),
)
if self.use_res_connect:
self.skip_drop = DropBlockScheduled(DropBlock2D(
drop_prob=drop_prob, block_size=7),
start_value=0.,
stop_value=drop_prob,
nr_steps=num_steps)
def __init__(self,
senet,
input3ch=True,
three_neck=False,
mixup_alpha=None,
mix_cand_layers=None,
use_mish=False,
cutmix_alpha=None,
cutmix_cand_layers=None,
output_layers=[2, 3, 4],
dropblock_p=None):
super(SENetEncoder_CalibMixup_Multiscale_v2, self).__init__()
self.three_neck = False
self.mixup_alpha = mixup_alpha
self.mix_cand_layers = mix_cand_layers
self.cutmix_alpha = cutmix_alpha
self.cutmix_cand_layers = cutmix_cand_layers
self.output_layers = output_layers
self.layer0 = senet.layer0
if not input3ch:
self.layer0 = first_conv2d_3ch_to_1ch(self.layer0)
self.layer1 = senet.layer1 # (256, 32, 32)
self.layer2 = senet.layer2 # (512, 16, 16)
self.layer3 = senet.layer3 # (1024, 8, 8)
self.layer4 = senet.layer4 # (2048, 4, 4)
self.calb_mixup0 = mixup.CalibrationMixup(layer_number=0)
self.calb_mixup1 = mixup.CalibrationMixup(layer_number=1)
self.calb_mixup2 = mixup.CalibrationMixup(layer_number=2)
self.calb_mixup3 = mixup.CalibrationMixup(layer_number=3)
self.dropblock0 = DropBlock2D(
drop_prob=dropblock_p,
block_size=10) if dropblock_p is not None else None
self.dropblock1 = DropBlock2D(
drop_prob=dropblock_p,
block_size=5) if dropblock_p is not None else None
self.dropblock2 = DropBlock2D(
drop_prob=dropblock_p,
block_size=5) if dropblock_p is not None else None
if use_mish:
mish.relu_to_mish(self.layer0)
mish.relu_to_mish(self.layer1)
mish.relu_to_mish(self.layer2)
mish.relu_to_mish(self.layer3)
mish.relu_to_mish(self.layer4)
self.pooling = nn.AdaptiveAvgPool2d(1)
def __init__(self, inplanes, planes, stride=1, downsample=None,
radix=2, cardinality=1, bottleneck_width=64,
avd=False, avd_first=False, dilation=1, is_first=False,
norm_layer=nn.BatchNorm2d, dropblock_prob=0.0, last_gamma=False):
super(Bottleneck, self).__init__()
group_width = int(planes * (bottleneck_width / 64.)) * cardinality
self.conv1 = nn.Conv2d(inplanes, group_width, kernel_size=1, bias=False)
self.bn1 = norm_layer(group_width)
self.dropblock_prob = dropblock_prob
self.radix = radix
self.avd = avd and (stride > 1 or is_first)
self.avd_first = avd_first
if self.avd:
self.avd_layer = nn.AvgPool2d(3, stride, padding=1)
stride = 1
if dropblock_prob > 0.0:
self.dropblock1 = DropBlock2D(dropblock_prob, 3)
if radix == 1:
self.dropblock2 = DropBlock2D(dropblock_prob, 3)
self.dropblock3 = DropBlock2D(dropblock_prob, 3)
if radix >= 1:
self.conv2 = SplAtConv2d(
group_width, group_width, kernel_size=3,
stride=stride, padding=dilation,
dilation=dilation, groups=cardinality, bias=False,
radix=radix,
norm_layer=norm_layer,
dropblock_prob=dropblock_prob)
else:
self.conv2 = nn.Conv2d(
group_width, group_width, kernel_size=3, stride=stride,
padding=dilation, dilation=dilation,
groups=cardinality, bias=False)
self.bn2 = norm_layer(group_width)
self.conv3 = nn.Conv2d(
group_width, planes * 4, kernel_size=1, bias=False)
self.bn3 = norm_layer(planes*4)
if last_gamma:
from torch.nn.init import zeros_
zeros_(self.bn3.weight)
self.relu = nn.ReLU(inplace=True)
self.downsample = downsample
self.dilation = dilation
self.stride = stride
def __init__(self,
in_channel,
D_filters,
drop_prob=0.1,
use_block=False,
block_size=5):
super().__init__()
self.D = D_filters
self.conv1 = nn.Conv2d(in_channel, self.D, 3, padding=1)
self.bn1 = nn.BatchNorm2d(self.D)
self.conv2 = nn.Conv2d(self.D, self.D, 3, padding=1)
self.bn2 = nn.BatchNorm2d(self.D)
self.conv3 = nn.Conv2d(self.D, self.D, 3, padding=1)
self.bn3 = nn.BatchNorm2d(self.D)
self.jump_conv = nn.Conv2d(in_channel, self.D, 1)
self.maxpool = nn.MaxPool2d(2, 2)
if use_block:
self.drop = DropBlock2D(block_size=block_size, drop_prob=drop_prob)
else:
self.drop = nn.Dropout(p=drop_prob)
nn.init.kaiming_normal_(self.conv1.weight)
nn.init.kaiming_normal_(self.conv2.weight)
nn.init.kaiming_normal_(self.conv3.weight)
nn.init.kaiming_normal_(self.jump_conv.weight)
def conv_bn(inp, out):
return nn.Sequential(
nn.Conv2d(inp, out, kernel_size=3, stride=1, padding=1, bias=False),
nn.BatchNorm2d(out),
DropBlock2D(block_size=3, drop_prob=0.1),
nn.PReLU(out)
)
def __init__(self,
in_ch,
out_ch,
circular_padding,
bilinear=True,
group_conv=False,
use_dropblock=False,
drop_p=0.5):
super(up, self).__init__()
# would be a nice idea if the upsampling could be learned too,
# but my machine do not have enough memory to handle all those weights
if bilinear:
self.up = nn.Upsample(scale_factor=2,
mode='bilinear',
align_corners=True)
elif group_conv:
self.up = nn.ConvTranspose2d(in_ch // 2,
in_ch // 2,
2,
stride=2,
groups=in_ch // 2)
else:
self.up = nn.ConvTranspose2d(in_ch // 2, in_ch // 2, 2, stride=2)
if circular_padding:
self.conv = double_conv_circular(in_ch,
out_ch,
group_conv=group_conv)
else:
self.conv = double_conv(in_ch, out_ch, group_conv=group_conv)
self.use_dropblock = use_dropblock
if self.use_dropblock:
self.dropblock = DropBlock2D(block_size=7, drop_prob=drop_p)
def __init__(self,
vgg_name='VGG19',
in_channel=3,
out_channel=10,
drop_prob=0.0,
block_size=5,
forward='dropblock'):
super(VGG2, self).__init__()
# 修改了 make_layer 这个部分， 源码是在上面， 把中间分层处理
self.f1, self.f2, self.f3 = self._make_layers2(vggcfg[vgg_name],
in_channel=in_channel)
self.classifier = nn.Linear(512, out_channel)
self.dropblock = LinearScheduler(DropBlock2D(drop_prob=drop_prob,
block_size=block_size),
start_value=0.,
stop_value=drop_prob,
nr_steps=5e3)
self.wh = 2
self.wh2 = 4
self.align_sche = False
self.i = 0
# self.cr = CropAndResize(8, 8) # 8 is according to the real size
self.cr = RoIAlign(self.wh, self.wh, transform_fpcoor=True)
self.cr2 = RoIAlign(self.wh2, self.wh2, transform_fpcoor=True)
# 注释掉 其中的一个，
if forward == 'dropblock':
self.forward = self._forward_dropblock
print("------- VGG with Dropblock ---------\n")
else:
self.forward = self._forward_align
print("------- VGG with ROiAlign ---------\n")
def __init__(self, feature_dim, with_norm='none', upsample_method='bilinear'):
super(FPT, self).__init__()
self.feature_dim = feature_dim
assert upsample_method in ['nearest', 'bilinear']
def interpolate(input):
return F.interpolate(input, scale_factor=2, mode=upsample_method, align_corners=False if upsample_method == 'bilinear' else None)
self.fpn_upsample = interpolate
assert with_norm in ['group_norm', 'batch_norm', 'none']
if with_norm == 'batch_norm':
norm = nn.BatchNorm2d
elif with_norm == 'group_norm':
def group_norm(num_channels):
return nn.GroupNorm(32, num_channels)
norm = group_norm
self.st_p5 = SelfTrans(n_head = 1, n_mix = 2, d_model = feature_dim, d_k= feature_dim, d_v= feature_dim)
self.st_p4 = SelfTrans(n_head = 1, n_mix = 2, d_model = feature_dim, d_k= feature_dim, d_v= feature_dim)
self.st_p3 = SelfTrans(n_head = 1, n_mix = 2, d_model = feature_dim, d_k= feature_dim, d_v= feature_dim)
self.st_p2 = SelfTrans(n_head = 1, n_mix = 2, d_model = feature_dim, d_k= feature_dim, d_v= feature_dim)
self.gt_p4_p5 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True)
self.gt_p3_p4 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True)
self.gt_p3_p5 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True)
self.gt_p2_p3 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True)
self.gt_p2_p4 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True)
self.gt_p2_p5 = GroundTrans(in_channels=feature_dim, inter_channels=None, mode='dot', dimension=2, bn_layer=True)
self.rt_p5_p4 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False)
self.rt_p5_p3 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False)
self.rt_p5_p2 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False)
self.rt_p4_p3 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False)
self.rt_p4_p2 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False)
self.rt_p3_p2 = RenderTrans(channels_high=feature_dim, channels_low=feature_dim, upsample=False)
drop_block = DropBlock2D(block_size=3, drop_prob=0.2)
if with_norm != 'none':
self.fpn_p5_1x1 = nn.Sequential(*[nn.Conv2d(2048, feature_dim, 1, bias=False), norm(feature_dim)])
self.fpn_p4_1x1 = nn.Sequential(*[nn.Conv2d(1024, feature_dim, 1, bias=False), norm(feature_dim)])
self.fpn_p3_1x1 = nn.Sequential(*[nn.Conv2d(512, feature_dim, 1, bias=False), norm(feature_dim)])
self.fpn_p2_1x1 = nn.Sequential(*[nn.Conv2d(256, feature_dim, 1, bias=False), norm(feature_dim)])
self.fpt_p5 = nn.Sequential(*[nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1, bias=False), norm(feature_dim)])
self.fpt_p4 = nn.Sequential(*[nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1, bias=False), norm(feature_dim)])
self.fpt_p3 = nn.Sequential(*[nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1, bias=False), norm(feature_dim)])
self.fpt_p2 = nn.Sequential(*[nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1, bias=False), norm(feature_dim)])
else:
self.fpn_p5_1x1 = nn.Conv2d(2048, feature_dim, 1)
self.fpn_p4_1x1 = nn.Conv2d(1024, feature_dim, 1)
self.fpn_p3_1x1 = nn.Conv2d(512, feature_dim, 1)
self.fpn_p2_1x1 = nn.Conv2d(256, feature_dim, 1)
self.fpt_p5 = nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1)
self.fpt_p4 = nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1)
self.fpt_p3 = nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1)
self.fpt_p2 = nn.Conv2d(feature_dim*5, feature_dim, 3, padding=1)
self.initialize()
def __init__(self, cfg, block, layers):
self.inplanes = 64
super(ResNet, self).__init__()
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False) #150*150
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.cfg = cfg
#----------new structure called DropBlock 11sr April-------------------------
if self.cfg.MODEL.BACKBONE.DROP_BLOCK:
drop_prob = 0.5
block_size = 3
self.dropblock = LinearScheduler(DropBlock2D(
drop_prob=drop_prob, block_size=block_size),
start_value=0.,
stop_value=drop_prob,
nr_steps=5)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) #75*75
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2) #38*38
self.layer3 = self._make_layer(block, 256, layers[2], stride=2) #19*19
#self.layer4 = self._make_layer(block, 512,layers[3] , stride=1) #10*10
self.ex_layer0 = self._make_layer(block, 512, 2, stride=2) #10*10
#self.maxpoo2 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
#2. extra_layers (ReLU will be used in the foward function) 10thApril,Xiaoyu Zhu
# if cfg.MODEL.BACKBONE.DEPTH>34:
# self.ex_layer1 = nn.Sequential(BasicBlock_modified(2048,512))
# #self.ex_layer1 = self._make_extra_layers(2048,512,3,1) #5*5
# else:
# self.ex_layer1 = nn.Sequential(BasicBlock_modified(512,512))
self.ex_layer1 = self._make_layer(block, 512, 1, stride=2) #5*5
#self.maxpoo3 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.ex_layer2 = self._make_layer(block, 256, 1, stride=2) #3*3
#self.maxpoo4 = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.ex_layer3 = self._make_layer(block, 128, 1, stride=2)
# if cfg.MODEL.BACKBONE.DEPTH>34:
# self.ex_layer3 = self._make_extra_layers(256*4,128,[2,3],0)
# else:
# #self.ex_layer3 = self._make_extra_layers(256,128,[2,3],0)
# self.ex_layer3 = self._make_layer(block, 128, 1, stride=2)
#BasicBlock_modified(inplanes=256, planes=128, kernel=[2,3],stride=2, padding=0)
# kaiming weight normal after default initialization
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def test_block_mask_hW_odd():
db = DropBlock2D(block_size=3, drop_prob=0.1)
mask = torch.tensor([[[0., 0., 0., 1., 0.], [0., 0., 0., 0., 0.],
[0., 0., 0., 0., 0.], [0., 0., 0., 0., 0.]]])
expected = torch.tensor([[[1., 1., 0., 0., 0.], [1., 1., 0., 0., 0.],
[1., 1., 1., 1., 1.], [1., 1., 1., 1., 1.]]])
block_mask = db._compute_block_mask(mask)
assert torch.equal(block_mask, expected)
def test_forward_pass2():
block_sizes = [2, 3, 4, 5, 6, 7, 8]
heights = [5, 6, 8, 10, 11, 14, 15]
widths = [5, 7, 8, 10, 15, 14, 15]
for block_size, height, width in zip(block_sizes, heights, widths):
dropout = DropBlock2D(0.1, block_size=block_size)
input = torch.randn((5, 20, height, width))
output = dropout(input)
assert tuple(input.shape) == tuple(output.shape)
def __init__(self, filters_in):
super(MSR_Convset_S, self).__init__()
self.__dw0 = Convolutional(filters_in=filters_in, filters_out=filters_in*2, kernel_size=3, stride=1,
pad=1, norm="bn", activate="leaky")
self.__pw0 = Convolutional(filters_in=filters_in*2, filters_out=filters_in, kernel_size=1, stride=1,
pad=0, norm="bn", activate="leaky")
self.__dw1 = Convolutional(filters_in=filters_in, filters_out=filters_in*2, kernel_size=3, stride=1,
pad=1, dila=1, norm="bn", activate="leaky")
self.__pw1 = Convolutional(filters_in=filters_in*2, filters_out=filters_in, kernel_size=1, stride=1,
pad=0, norm="bn", activate="leaky")
self.__drop = LinearScheduler(DropBlock2D(block_size=3, drop_prob=0.1), start_value=0., stop_value=0.1, nr_steps=5)
def __init__(self,
in_channels,
channels,
kernel_size,
stride=(1, 1),
padding=(0, 0),
dilation=(1, 1),
groups=1,
bias=True,
radix=2,
reduction_factor=4,
rectify=False,
rectify_avg=False,
norm_layer=None,
dropblock_prob=0.0,
**kwargs):
super(SplAtConv2d, self).__init__()
padding = _pair(padding)
self.rectify = rectify and (padding[0] > 0 or padding[1] > 0)
self.rectify_avg = rectify_avg
inter_channels = max(in_channels * radix // reduction_factor, 32)
self.radix = radix
self.cardinality = groups
self.channels = channels
self.dropblock_prob = dropblock_prob
self.conv = nn.Conv2d(in_channels,
channels * radix,
kernel_size,
stride,
padding,
dilation,
groups=groups * radix,
bias=bias,
**kwargs)
self.use_bn = norm_layer is not None
if self.use_bn:
self.bn0 = norm_layer(channels * radix)
self.relu = nn.ReLU(inplace=True)
self.fc1 = nn.Conv2d(channels,
inter_channels,
1,
groups=self.cardinality)
if self.use_bn:
self.bn1 = norm_layer(inter_channels)
self.fc2 = nn.Conv2d(inter_channels,
channels * radix,
1,
groups=self.cardinality)
if dropblock_prob > 0.0:
self.dropblock = DropBlock2D(dropblock_prob, 3)
self.rsoftmax = rSoftMax(radix, groups)
def __init__(self,
block,
layers,
num_classes=1000,
drop_prob=0.,
block_size=5):
super(ResNet, self).__init__()
self.inplanes = 64
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.dropblock = LinearScheduler(DropBlock2D(drop_prob=drop_prob,
block_size=block_size),
start_value=0.,
stop_value=drop_prob,
nr_steps=5e3)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
self.fc = nn.Linear(512 * block.expansion, num_classes)
self.wh = 2
self.wh2 = 8
self.align_sche = False
self.i = 0
# self.cr = CropAndResize(8, 8) # 8 is according to the real size
self.cr = RoIAlign(self.wh, self.wh, transform_fpcoor=True)
self.cr2 = RoIAlign(self.wh2, self.wh2, transform_fpcoor=True)
print("--------------------------------------------------------"
"\n-------- RoiAlign -------\n"
"--------------------------------------------------------")
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
def __init__(self, block, num_blocks, num_classes=10, drop_prob=0., block_size=5):
super(ResNet, self).__init__()
self.in_planes = 64
self.conv1 = conv3x3(3, 64)
self.bn1 = nn.BatchNorm2d(64)
self.dropblock = LinearScheduler(
DropBlock2D(drop_prob=drop_prob, block_size=block_size, att=True),
start_value=0.,
stop_value=drop_prob,
nr_steps=5e4
)
self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
self.linear = nn.Linear(512 * block.expansion, num_classes)
def __init__(self,
block,
layers,
num_classes=1000,
drop_prob=0.,
block_size=5,
gkernel=False):
super(ResNet, self).__init__()
self.inplanes = 64
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.dropblock = LinearScheduler(DropBlock2D(drop_prob=drop_prob,
block_size=block_size,
gkernel=gkernel),
start_value=0.,
stop_value=drop_prob,
nr_steps=5e3)
self.dropcblock = LinearScheduler(DropCBlock(drop_prob=drop_prob,
block_size=block_size),
start_value=0.,
stop_value=drop_prob,
nr_steps=5e3)
# self.dropchannel = DropChannel(drop_prob=drop_prob)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
self.fc = nn.Linear(512 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
def __init__(self, input_channels, input_width, num_classes=-1, dropout_prob=0, drop_prob=0, block_size=5): super(SimpleCNN, self).__init__() if drop_prob != 0: self.dropblock = LinearScheduler( DropBlock2D(drop_prob=drop_prob, block_size=block_size), start_value=0., stop_value=drop_prob, nr_steps=5e3 ) self.drop_prob = drop_prob self.block_size = block_size self.dropout_prob = dropout_prob self.dim = input_width self.conv1 = nn.Conv2d(input_channels, 50, kernel_size=3) self.conv2 = nn.Conv2d(50, 100, kernel_size=5) self.conv3 = nn.Conv2d(100, 200, kernel_size=5) self.conv4 = nn.Conv2d(200, 400, kernel_size=2) self.fc1 = nn.Linear(400, 300) self.fc2 = nn.Linear(300, num_classes)
def __init__(self,
in_planes,
out_planes,
stride,
dropRate=0.0,
my_conv=Conv2d):
super(BasicBlock, self).__init__()
self.bn1 = nn.BatchNorm2d(in_planes)
# self.bn1 = nn.GroupNorm(num_groups=8,num_channels=in_planes)
# self.relu1 = nn.ReLU(inplace=True)
self.relu1 = Mish()
self.conv1 = my_conv(in_planes,
out_planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False)
self.bn2 = nn.BatchNorm2d(out_planes)
# self.bn2 = nn.GroupNorm(num_groups=8,num_channels=out_planes)
# self.relu2 = nn.ReLU(inplace=True)
self.relu2 = Mish()
self.conv2 = my_conv(out_planes,
out_planes,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.droprate = dropRate
from dropblock import LinearScheduler, DropBlock2D
self.conv2_drop = LinearScheduler(DropBlock2D(drop_prob=dropRate,
block_size=3),
start_value=0,
stop_value=dropRate,
nr_steps=10000)
self.equalInOut = (in_planes == out_planes)
self.convShortcut = (not self.equalInOut) and my_conv(
in_planes,
out_planes,
kernel_size=1,
stride=stride,
padding=0,
bias=False) or None
def test_forward_pass():
db = DropBlock2D(block_size=3, drop_prob=0.1)
block_mask = torch.tensor([[[0., 0., 0., 1., 1., 1., 1.],
[0., 0., 0., 0., 0., 0., 1.],
[0., 0., 0., 0., 0., 0., 1.],
[1., 1., 1., 0., 0., 0., 1.],
[1., 1., 1., 1., 1., 1., 1.],
[1., 1., 1., 1., 1., 1., 1.],
[1., 1., 1., 1., 1., 1., 1.]]])
db._compute_block_mask = mock.MagicMock(return_value=block_mask)
x = torch.ones(10, 10, 7, 7)
h = db(x)
expected = block_mask * block_mask.numel() / block_mask.sum()
expected = expected[:, None, :, :].expand_as(x)
assert tuple(h.shape) == (10, 10, 7, 7)
assert torch.equal(h, expected)
def __init__(self, cfg, block, layers):
self.inplanes = 64
super(ResNet, self).__init__()
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False) #150*150
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.cfg = cfg
#----------new structure called DropBlock 11sr April-------------------------
if self.cfg.MODEL.BACKBONE.DROP_BLOCK:
drop_prob = 0.5
block_size = 3
self.dropblock = LinearScheduler(DropBlock2D(
drop_prob=drop_prob, block_size=block_size),
start_value=0.,
stop_value=drop_prob,
nr_steps=5)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2) #75*75
self.layer3 = self._make_layer(block, 256, layers[2], stride=2) #38*38
#2. extra_layers (ReLU will be used in the foward function) 10thApril,Xiaoyu Zhu
self.ex_layer00 = self._make_layer(block, 512, 1, stride=2) #19*19
self.ex_layer0 = self._make_layer(block, 256, 1, stride=2) #10*10
self.ex_layer1 = self._make_layer(block, 256, 1, stride=2) #5*5
self.ex_layer2 = self._make_layer(block, 128, 1, stride=2) #3*3
self.ex_layer3 = self._make_layer(block, 128, 1, stride=2) #1*2
# kaiming weight normal after default initialization
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def __init__(self, cfg, block, layers):
self.inplanes = 64
super(ResNet, self).__init__()
self.conv1 = nn.Conv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False) #150*150
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.cfg = cfg
#----------new structure called DropBlock 11sr April-------------------------
if self.cfg.MODEL.BACKBONE.DROP_BLOCK:
drop_prob = 0.
block_size = 5
self.dropblock = LinearScheduler(DropBlock2D(
drop_prob=drop_prob, block_size=block_size),
start_value=0.,
stop_value=drop_prob,
nr_steps=5)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) #75*75
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2) #38*38
self.layer3 = self._make_layer(block, 256, layers[2], stride=2) #19*19
self.layer4 = self._make_layer(block, 512, layers[3], stride=2) #10*10
#2. extra_layers (ReLU will be used in the foward function) 10thApril,Xiaoyu Zhu
if cfg.MODEL.BACKBONE.DEPTH > 34:
self.ex_layer1 = self._make_extra_layers(2048, 512, 3, 1) #5*5
else:
self.ex_layer1 = self._make_extra_layers(512, 512, 3, 1) #5*5
self.ex_layer2 = self._make_extra_layers(512, 256, 3, 1) #3*3
self.ex_layer3 = self._make_extra_layers(256, 128, [2, 3], 0) #1*1
#-----------------------------------Old Version-------------------
# 2. extra_layers (ReLU will be used in the foward function)
# extra_layer=[] #This is the old_version of the arch. But I want to add some batch_norm in to the layers
# if cfg.MODEL.BACKBONE.DEPTH>34:
# extra_layer.append(torch.nn.Conv2d(in_channels=2048, out_channels=256,kernel_size=3,stride=1,padding=1))
# else:
# extra_layer.append(torch.nn.Conv2d(in_channels=512, out_channels=256,kernel_size=3,stride=1,padding=1))
# #need ReLU
# extra_layer.append(torch.nn.Conv2d(in_channels=256, out_channels=512,kernel_size=3,stride=2,padding=1)) #5*5
# #need ReLU
# extra_layer.append(torch.nn.Conv2d(in_channels=512, out_channels=512,kernel_size=3,stride=1,padding=1))
# #need ReLU
# extra_layer.append(torch.nn.Conv2d(in_channels=512, out_channels=256,kernel_size=3,stride=2,padding=1)) #3*3
# #need ReLU
# extra_layer.append(torch.nn.Conv2d(in_channels=256, out_channels=256,kernel_size=3,stride=1,padding=1))
# #need ReLU
# extra_layer.append(torch.nn.Conv2d(in_channels=256, out_channels=128,kernel_size=3,stride=2,padding=0)) #1*1
# #need ReLU
# self.extra_layer=torch.nn.Sequential(*extra_layer)
#-----------------------------------------------------------------
# kaiming weight normal after default initialization
for m in self.modules():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def __init__(self,
senet,
input3ch=True,
three_neck=False,
mixup_alpha=None,
mix_cand_layers=None,
use_mish=False,
cutmix_alpha=None,
cutmix_cand_layers=None,
output_layer3=False,
output_layer2=False,
dropblock_p=None,
upsample_size=None,
calib_mixup=False):
super(SENetEncoder_Mixup, self).__init__()
self.three_neck = three_neck
self.mixup_alpha = mixup_alpha
self.mix_cand_layers = mix_cand_layers
self.cutmix_alpha = cutmix_alpha
self.cutmix_cand_layers = cutmix_cand_layers
self.output_layer3 = output_layer3
self.output_layer2 = output_layer2
self.layer0 = senet.layer0
if not input3ch:
self.layer0 = first_conv2d_3ch_to_1ch(self.layer0)
self.layer1 = senet.layer1
self.layer2 = senet.layer2
self.layer3 = senet.layer3
if three_neck:
self.layer4_1 = copy_model(senet.layer4)
self.layer4_2 = copy_model(senet.layer4)
self.layer4_3 = copy_model(senet.layer4)
else:
self.layer4 = copy_model(senet.layer4)
self.calb_mixup0 = mixup.CalibrationMixup(
layer_number=0) if calib_mixup else None
self.calb_mixup1 = mixup.CalibrationMixup(
layer_number=1) if calib_mixup else None
self.calb_mixup2 = mixup.CalibrationMixup(
layer_number=2) if calib_mixup else None
self.calb_mixup3 = mixup.CalibrationMixup(
layer_number=3) if calib_mixup else None
self.dropblock0 = DropBlock2D(
drop_prob=dropblock_p,
block_size=10) if dropblock_p is not None else None
self.dropblock1 = DropBlock2D(
drop_prob=dropblock_p,
block_size=5) if dropblock_p is not None else None
self.dropblock2 = DropBlock2D(
drop_prob=dropblock_p,
block_size=5) if dropblock_p is not None else None
self.upsample = nn.Upsample(
size=upsample_size, mode='bilinear',
align_corners=True) if upsample_size is not None else None
if use_mish:
mish.relu_to_mish(self.layer0)
mish.relu_to_mish(self.layer1)
mish.relu_to_mish(self.layer2)
mish.relu_to_mish(self.layer3)
if three_neck:
mish.relu_to_mish(self.layer4_1)
mish.relu_to_mish(self.layer4_2)
mish.relu_to_mish(self.layer4_3)
else:
mish.relu_to_mish(self.layer4)
def test_large_block_size():
dropout = DropBlock2D(0.3, block_size=9)
x = torch.rand(100, 10, 16, 16)
output = dropout(x)
assert tuple(x.shape) == tuple(output.shape)
def __init__(self, num_classes=1000, drop_prob=0.3, block_size=3):
""" Constructor
Args:
num_classes: number of classes
"""
super(Xception_drop, self).__init__()
self.num_classes = num_classes
self.conv1 = nn.Conv2d(3, 32, 3, 2, 0, bias=False)
self.bn1 = nn.BatchNorm2d(32)
self.relu1 = nn.ReLU(inplace=True)
self.conv2 = nn.Conv2d(32, 64, 3, bias=False)
self.bn2 = nn.BatchNorm2d(64)
self.relu2 = nn.ReLU(inplace=True)
#do relu here
self.dropblock = LinearScheduler(DropBlock2D(drop_prob=drop_prob,
block_size=block_size),
start_value=0.,
stop_value=drop_prob,
nr_steps=int(5e3))
self.block1 = Block(64,
128,
2,
2,
start_with_relu=False,
grow_first=True)
self.block2 = Block(128,
256,
2,
2,
start_with_relu=True,
grow_first=True)
self.block3 = Block(256,
728,
2,
2,
start_with_relu=True,
grow_first=True)
self.block4 = Block(728,
728,
3,
1,
start_with_relu=True,
grow_first=True)
self.block5 = Block(728,
728,
3,
1,
start_with_relu=True,
grow_first=True)
self.block6 = Block(728,
728,
3,
1,
start_with_relu=True,
grow_first=True)
self.block7 = Block(728,
728,
3,
1,
start_with_relu=True,
grow_first=True)
self.block8 = Block(728,
728,
3,
1,
start_with_relu=True,
grow_first=True)
self.block9 = Block(728,
728,
3,
1,
start_with_relu=True,
grow_first=True)
self.block10 = Block(728,
728,
3,
1,
start_with_relu=True,
grow_first=True)
self.block11 = Block(728,
728,
3,
1,
start_with_relu=True,
grow_first=True)
self.block12 = Block(728,
1024,
2,
2,
start_with_relu=True,
grow_first=False)
self.conv3 = SeparableConv2d(1024, 1536, 3, 1, 1)
self.bn3 = nn.BatchNorm2d(1536)
self.relu3 = nn.ReLU(inplace=True)
#do relu here
self.conv4 = SeparableConv2d(1536, 2048, 3, 1, 1)
self.bn4 = nn.BatchNorm2d(2048)
self.fc = nn.Linear(2048, num_classes)
def test_forward_pass_with_cuda():
dropout = DropBlock2D(0.3, block_size=5).to('cuda')
x = torch.rand(100, 10, 16, 16).to('cuda')
output = dropout(x)
assert tuple(x.shape) == tuple(output.shape)
def __init__(self, output_class=7, model_path=None, resnet_type=34, drop_block=False, drop_prob=0.5, drop_pos=None, layer_depth=4, drop_block_size=7):
super(ResNetDropblock, self).__init__()
assert resnet_type in [18, 34, 50]
assert layer_depth in [1, 2, 3, 4]
if resnet_type == 18:
self.base = resnet18(pretrained=False, num_classes=1000)
# state_dict = torch.load('resnet18.pth')
if layer_depth == 4:
last_fc_in_channel = 512 * 1
elif layer_depth == 3:
last_fc_in_channel = 256 * 1
elif layer_depth == 2:
last_fc_in_channel = 128 * 1
else: # elif layer_depth == 1:
last_fc_in_channel = 64 * 1
elif resnet_type == 34:
self.base = resnet34(pretrained=False, num_classes=1000)
# state_dict = torch.load('resnet34.pth')
if layer_depth == 4:
last_fc_in_channel = 512 * 1
elif layer_depth == 3:
last_fc_in_channel = 256 * 1
elif layer_depth == 2:
last_fc_in_channel = 128 * 1
else: # elif layer_depth == 1:
last_fc_in_channel = 64 * 1
else: # elif resnet_type == 50:
self.base = resnet50(pretrained=False, num_classes=1000)
# state_dict = torch.load('resnet50.pth')
if layer_depth == 4:
last_fc_in_channel = 512 * 4
elif layer_depth == 3:
last_fc_in_channel = 256 * 4
elif layer_depth == 2:
last_fc_in_channel = 128 * 4
else: # elif layer_depth == 1:
last_fc_in_channel = 64 * 4
# self.base.load_state_dict(state_dict)
# def weight_init(m):
# if isinstance(m, nn.Conv2d) or isinstance(m, nn.ConvTranspose2d):
# nn.init.xavier_uniform_(m.weight, gain=nn.init.calculate_gain('relu'))
# if m.bias is not None:
# nn.init.zeros_(m.bias)
#
# self.base.layer3.apply(weight_init)
# self.base.layer4.apply(weight_init)
self.base.fc = nn.Linear(in_features=last_fc_in_channel, out_features=output_class, bias=True)
if drop_block:
self.dropblock = LinearScheduler(
DropBlock2D(drop_prob=drop_prob, block_size=drop_block_size),
start_value=0.,
stop_value=drop_prob,
nr_steps=300
)
else:
self.dropblock = nn.Sequential()
self.drop_pos = drop_pos
self.layer_depth = layer_depth
if model_path is not None:
self.model_path = model_path
assert '.pth' in self.model_path or '.pkl' in self.model_path
self.init_weight()
def __init__(
self,
block,
layers,
groups,
reduction,
dropout_p=0.2,
inplanes=128,
input_3x3=True,
downsample_kernel_size=3,
downsample_padding=1,
num_classes=1000,
strides=[
2,
] * 5,
adaptive_pool=False,
input_c=3,
):
"""
Parameters
----------
block (nn.Module): Bottleneck class.
- For SENet154: SEBottleneck
- For SE-ResNet models: SEResNetBottleneck
- For SE-ResNeXt models: SEResNeXtBottleneck
layers (list of ints): Number of residual blocks for 4 layers of the
network (layer1...layer4).
groups (int): Number of groups for the 3x3 convolution in each
bottleneck block.
- For SENet154: 64
- For SE-ResNet models: 1
- For SE-ResNeXt models: 32
reduction (int): Reduction ratio for Squeeze-and-Excitation modules.
- For all models: 16
dropout_p (float or None): Drop probability for the Dropout layer.
If `None` the Dropout layer is not used.
- For SENet154: 0.2
- For SE-ResNet models: None
- For SE-ResNeXt models: None
inplanes (int): Number of input channels for layer1.
- For SENet154: 128
- For SE-ResNet models: 64
- For SE-ResNeXt models: 64
input_3x3 (bool): If `True`, use three 3x3 convolutions instead of
a single 7x7 convolution in layer0.
- For SENet154: True
- For SE-ResNet models: False
- For SE-ResNeXt models: False
downsample_kernel_size (int): Kernel size for downsampling convolutions
in layer2, layer3 and layer4.
- For SENet154: 3
- For SE-ResNet models: 1
- For SE-ResNeXt models: 1
downsample_padding (int): Padding for downsampling convolutions in
layer2, layer3 and layer4.
- For SENet154: 1
- For SE-ResNet models: 0
- For SE-ResNeXt models: 0
num_classes (int): Number of outputs in `last_linear` layer.
- For all models: 1000
"""
super(SENet, self).__init__()
self.inplanes = inplanes
if input_3x3:
layer0_modules = [
('conv1',
nn.Conv2d(input_c,
64,
3,
stride=strides[0],
padding=1,
bias=False)),
('bn1', nn.BatchNorm2d(64)),
('relu1', nn.ReLU(inplace=True)),
('conv2', nn.Conv2d(64, 64, 3, stride=1, padding=1,
bias=False)),
('bn2', nn.BatchNorm2d(64)),
('relu2', nn.ReLU(inplace=True)),
('conv3',
nn.Conv2d(64, inplanes, 3, stride=1, padding=1, bias=False)),
('bn3', nn.BatchNorm2d(inplanes)),
('relu3', nn.ReLU(inplace=True)),
]
else:
layer0_modules = [
('conv1',
nn.Conv2d(input_c,
inplanes,
kernel_size=7,
stride=strides[0],
padding=3,
bias=False)),
('bn1', nn.BatchNorm2d(inplanes)),
('relu1', nn.ReLU(inplace=True)),
]
# To preserve compatibility with Caffe weights `ceil_mode=True`
# is used instead of `padding=1`.
layer0_modules.append(
('pool', nn.MaxPool2d(3, stride=strides[1], ceil_mode=True)))
self.layer0 = nn.Sequential(OrderedDict(layer0_modules))
self.layer1 = self._make_layer(block,
planes=64,
blocks=layers[0],
groups=groups,
reduction=reduction,
downsample_kernel_size=1,
downsample_padding=0)
self.layer2 = self._make_layer(
block,
planes=128,
blocks=layers[1],
stride=strides[2],
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding)
self.layer3 = self._make_layer(
block,
planes=256,
blocks=layers[2],
stride=strides[3],
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding)
self.layer4 = self._make_layer(
block,
planes=512,
blocks=layers[3],
stride=strides[4],
groups=groups,
reduction=reduction,
downsample_kernel_size=downsample_kernel_size,
downsample_padding=downsample_padding)
self.dropblock0 = DropBlock2D(drop_prob=0.2, block_size=16)
self.dropblock1 = DropBlock2D(drop_prob=0.2, block_size=8)
if adaptive_pool:
self.avg_pool = nn.AdaptiveAvgPool2d(output_size=1)
else:
self.avg_pool = nn.AvgPool2d(7, stride=1)
self.dropout = nn.Dropout(dropout_p) if dropout_p is not None else None
self.last_linear = nn.Linear(512 * block.expansion, num_classes)
def __init__(self, num_class=11):
super(Resnet50Mod, self).__init__()
self.cnn = nn.Sequential(*list(origin_net.children())[:-1])
self.hidden_layer = nn.Linear(2048, 128)
self.dropout = DropBlock2D(block_size=3, drop_prob=0.2)
self.output = nn.Linear(128, num_class) # 11 or 10, 1 * 11
