def __init__(self, C_prev_prev, C_prev, C):
super().__init__()
self.reduction = True
self.preprocess0 = ReLUConvBN(C_prev_prev, C, 1)
self.preprocess1 = ReLUConvBN(C_prev, C, 1)
self.branch_a1 = Sequential([
Act(),
Conv2d(C, C, (1, 3), stride=(1, 2), groups=8, bias=False),
Conv2d(C, C, (3, 1), stride=(2, 1), groups=8, bias=False),
Norm(C, affine=True),
Act(),
Conv2d(C, C, 1),
Norm(C, affine=True),
])
self.branch_a2 = Sequential([
Pool2d(3, stride=2, type='max'),
Norm(C, affine=True),
])
self.branch_b1 = Sequential([
Act(),
Conv2d(C, C, (1, 3), stride=(1, 2), groups=8, bias=False),
Conv2d(C, C, (3, 1), stride=(2, 1), groups=8, bias=False),
Norm(C, affine=True),
Act(),
Conv2d(C, C, 1),
Norm(C, affine=True),
])
self.branch_b2 = Sequential([
Pool2d(3, stride=2, type='max'),
Norm(C, affine=True),
])
def __init__(self, in_channels, channels, stride, channels_per_group, cardinality,
start_block=False, end_block=False, exclude_bn0=False):
super().__init__()
out_channels = channels * self.expansion
width = channels_per_group * cardinality
if not start_block and not exclude_bn0:
self.bn0 = Norm(in_channels)
if not start_block:
self.act0 = Act()
self.conv1 = Conv2d(in_channels, width, kernel_size=1)
self.bn1 = Norm(width)
self.act1 = Act()
self.conv2 = Conv2d(width, width, kernel_size=3, stride=stride, groups=cardinality,
norm='def', act='def')
self.conv3 = Conv2d(width, out_channels, kernel_size=1)
if start_block:
self.bn3 = Norm(out_channels)
if end_block:
self.bn3 = Norm(out_channels)
self.act3 = Act()
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(3, 2, type='max'))
shortcut.append(
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.start_block = start_block
self.end_block = end_block
self.exclude_bn0 = exclude_bn0
def __init__(self,
in_channels,
out_channels,
stride,
dropout,
use_se=False):
super().__init__()
self.use_se = use_se
self.norm1 = Norm(in_channels)
self.act1 = Act()
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=stride)
self.norm2 = Norm(out_channels)
self.act2 = Act()
self.dropout = Dropout(dropout) if dropout else Identity()
self.conv2 = Conv2d(out_channels, out_channels, kernel_size=3)
if self.use_se:
self.se = SELayer(out_channels, reduction=8)
if stride != 1:
assert in_channels != out_channels
self.shortcut = Sequential([
Pool2d(2, 2, type='avg'),
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'),
])
else:
self.shortcut = Identity()
def __init__(self, in_channels, out_channels, dropout):
layers = [
Norm(in_channels),
Act(),
Conv2d(in_channels, out_channels, kernel_size=3),
Norm(out_channels),
Act(),
Conv2d(out_channels, out_channels, kernel_size=3),
]
if dropout:
layers.insert(5, Dropout(dropout))
super().__init__(layers)
def __init__(self,
in_channels,
channels,
stride,
dropout=0,
drop_path=0,
avd=False,
start_block=False,
end_block=False,
exclude_bn0=False):
super().__init__()
out_channels = channels * self.expansion
if not start_block and not exclude_bn0:
self.bn0 = Norm(in_channels)
if not start_block:
self.act0 = Act()
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=stride)
self.bn1 = Norm(out_channels)
self.act1 = Act()
self.dropout = Dropout(dropout) if dropout else Identity()
self.conv2 = Conv2d(out_channels, out_channels, kernel_size=3)
if start_block:
self.bn2 = Norm(out_channels)
self.drop_path = DropPath(drop_path) if drop_path else Identity()
if end_block:
self.bn2 = Norm(out_channels)
self.act2 = Act()
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(3, 2, type='max'))
if in_channels != out_channels:
shortcut.append(
Conv2d(in_channels,
out_channels,
kernel_size=1,
norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.start_block = start_block
self.end_block = end_block
self.exclude_bn0 = exclude_bn0
def __init__(self, in_channels, out_channels, dropout, reduction):
layers = [
Norm(in_channels),
Act(),
Conv2d(in_channels, out_channels, kernel_size=3),
Norm(out_channels),
Act(),
Conv2d(out_channels, out_channels, kernel_size=3),
]
if dropout:
layers.insert(5, Dropout(dropout))
layers.append(SELayer(out_channels, reduction=reduction))
super().__init__(layers)
def __init__(self,
in_channels,
channels,
stride,
base_width,
scale,
cardinality,
end_block=False,
exclude_bn0=False):
super().__init__()
out_channels = channels * self.expansion
start_block = stride != 1 or in_channels != out_channels
width = math.floor(channels * (base_width / 64)) * scale * cardinality
if not start_block and not exclude_bn0:
self.bn0 = Norm(in_channels)
if not start_block:
self.act0 = Act()
self.conv1 = Conv2d(in_channels, width, kernel_size=1)
self.bn1 = Norm(width)
self.act1 = Act()
self.conv2 = Res2Conv(width,
width,
kernel_size=3,
stride=stride,
scale=scale,
groups=cardinality,
norm='def',
act='def',
start_block=start_block)
self.conv3 = Conv2d(width, out_channels, kernel_size=1)
if start_block:
self.bn3 = Norm(out_channels)
if end_block:
self.bn3 = Norm(out_channels)
self.act3 = Act()
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(2, 2, type='avg'))
shortcut.append(
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.start_block = start_block
self.end_block = end_block
self.exclude_bn0 = exclude_bn0
def __init__(self, in_channels, out_channels, stride, groups, reduction,
zero_init_residual):
super().__init__()
se_channels = in_channels // reduction
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=1,
norm='def',
act='def')
self.conv2 = Conv2d(out_channels,
out_channels,
kernel_size=3,
stride=stride,
groups=groups,
norm='def',
act='def')
self.se = SELayer(out_channels, se_channels)
self.conv3 = Sequential([
Conv2d(out_channels, out_channels, kernel_size=1, bias=False),
Norm(out_channels,
gamma_init='zeros' if zero_init_residual else 'ones')
])
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(2, 2, type='avg'))
shortcut.append(
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.act = Act()
def __init__(self,
in_channels,
channels,
stride,
zero_init_residual=True,
reduction=16):
super().__init__()
out_channels = channels * self.expansion
self.conv1 = Conv2d(in_channels,
channels,
kernel_size=1,
norm='def',
act='def')
self.conv2 = Conv2d(channels,
channels,
kernel_size=3,
stride=stride,
norm='def',
act='def')
self.conv3 = Conv2d(channels, out_channels, kernel_size=1)
self.bn3 = Norm(out_channels,
gamma_init='zeros' if zero_init_residual else 'ones')
self.se = SELayer(out_channels, reduction=reduction)
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(2, 2, type='avg'))
shortcut.append(
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.act = Act()
def __init__(self, in_channels, out_channels, dropout, use_se, drop_path):
layers = [
Norm(in_channels),
Act(),
Conv2d(in_channels, out_channels, kernel_size=3),
Norm(out_channels),
Act(),
Conv2d(out_channels, out_channels, kernel_size=3),
]
if dropout:
layers.insert(5, Dropout(dropout))
if use_se:
layers.append(SELayer(out_channels, reduction=8))
if drop_path:
layers.append(DropPath(drop_path))
super().__init__(layers)
def __init__(self, in_channels, out_channels, stride, groups, use_se):
super().__init__()
self.use_se = use_se
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=1,
norm='def',
act='def')
self.conv2 = Conv2d(out_channels,
out_channels,
kernel_size=3,
stride=stride,
groups=groups,
norm='def',
act='def')
if self.use_se:
self.se = SELayer(out_channels, 4)
self.conv3 = Sequential(
Conv2d(out_channels, out_channels, kernel_size=1, bias=False),
Norm(out_channels, gamma_init='zeros'))
if stride != 1 or in_channels != out_channels:
self.shortcut = Conv2d(in_channels,
out_channels,
kernel_size=1,
norm='def')
else:
self.shortcut = Identity()
self.act = Act()
def __init__(self, in_channels, channels, stride, base_width, splits, zero_init_residual, genotype):
super().__init__()
self.stride = stride
out_channels = channels * self.expansion
width = math.floor(out_channels // self.expansion * (base_width / 64)) * splits
self.conv1 = Conv2d(in_channels, width, kernel_size=1,
norm='def', act='def')
if stride == 1:
self.conv2 = PPConv(width, splits=splits, genotype=genotype)
else:
self.conv2 = Conv2d(width, width, kernel_size=3, stride=2, groups=splits,
norm='def', act='def')
self.conv3 = Conv2d(width, out_channels, kernel_size=1)
self.bn3 = Norm(out_channels, gamma_init='zeros' if zero_init_residual else 'ones')
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(2, 2, type='avg'))
shortcut.append(
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.act = Act()
def __init__(self, in_channels, channels, stride, cardinality, base_width):
super().__init__()
out_channels = channels * 4
D = math.floor(channels * (base_width / 64))
C = cardinality
self.conv1 = Conv2d(in_channels,
D * C,
kernel_size=1,
norm='def',
act='def')
self.conv2 = Conv2d(D * C,
D * C,
kernel_size=3,
stride=stride,
groups=cardinality,
norm='def',
act='def')
self.conv3 = Conv2d(D * C, out_channels, kernel_size=1, norm='def')
self.shortcut = Conv2d(
in_channels,
out_channels,
kernel_size=1,
stride=stride,
norm='def') if in_channels != out_channels else Identity()
self.act = Act()
def __init__(self, depth, k, dropout=0, reduction=8, num_classes=10):
super().__init__()
num_blocks = (depth - 4) // 6
self.conv = Conv2d(3, self.stages[0], kernel_size=3)
self.layer1 = self._make_layer(self.stages[0] * 1,
self.stages[1] * k,
num_blocks,
stride=1,
dropout=dropout,
reduction=reduction)
self.layer2 = self._make_layer(self.stages[1] * k,
self.stages[2] * k,
num_blocks,
stride=2,
dropout=dropout,
reduction=reduction)
self.layer3 = self._make_layer(self.stages[2] * k,
self.stages[3] * k,
num_blocks,
stride=2,
dropout=dropout,
reduction=reduction)
self.norm = Norm(self.stages[3] * k)
self.act = Act()
self.avgpool = GlobalAvgPool()
self.fc = Linear(self.stages[3] * k, num_classes)
def __init__(self,
in_channels,
out_channels=256,
num_extra_convs=2,
extra_convs_on='input',
norm='bn'):
super().__init__()
assert isinstance(in_channels, list)
self.in_channels = in_channels
self.out_channels = out_channels
self.num_extra_convs = num_extra_convs
assert extra_convs_on in ['input', 'output']
self.extra_convs_on = extra_convs_on
self.lateral_convs = []
self.fpn_convs = []
for in_channels in self.in_channels:
self.lateral_convs.append(
Conv2d(in_channels, out_channels, 1))
self.fpn_convs.append(
Conv2d(out_channels, out_channels, 3, norm=norm))
self.extra_convs = []
in_channels = self.in_channels[-1] if extra_convs_on == 'input' else out_channels
for i in range(num_extra_convs):
extra_conv = Conv2d(in_channels, out_channels, 3, stride=2, norm=norm)
if i != 0:
extra_conv = Sequential([Act(), extra_conv])
self.extra_convs.append(extra_conv)
in_channels = out_channels
num_levels = len(self.in_channels) + num_extra_convs
self.feat_channels = [out_channels] * num_levels
def __init__(self, in_channels, channels, stride, dilation=1, base_width=26, scale=4,
zero_init_residual=True, avd=False):
super().__init__()
out_channels = channels * self.expansion
start_block = stride != 1 or in_channels != out_channels
width = math.floor(channels * (base_width / 64)) * scale
self.conv1 = Conv2d(in_channels, width, kernel_size=1,
norm='def', act='def')
if avd and stride != 1:
self.conv2 = Sequential([
Pool2d(3, stride=stride, type='avg'),
Res2Conv(width, width, kernel_size=3, stride=1, dilation=dilation, scale=scale,
groups=1, start_block=start_block, norm='def', act='def'),
])
else:
self.conv2 = Res2Conv(width, width, kernel_size=3, stride=stride, dilation=dilation,
scale=scale, groups=1, start_block=start_block, norm='def', act='def')
self.conv3 = Conv2d(width, out_channels, kernel_size=1)
self.bn3 = Norm(out_channels, gamma_init='zeros' if zero_init_residual else 'ones')
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(2, 2, type='avg'))
shortcut.append(
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.act = Act()
def __init__(self, in_channels, out_channels, stride, groups):
super().__init__()
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=1,
norm='def',
act='def')
self.conv2 = Conv2d(out_channels,
out_channels,
kernel_size=3,
stride=stride,
groups=groups,
norm='def',
act='def')
self.conv3 = Sequential([
Conv2d(out_channels, out_channels, kernel_size=1, bias=False),
Norm(out_channels)
])
self.eca = ECALayer()
if stride != 1 or in_channels != out_channels:
self.shortcut = Conv2d(in_channels,
out_channels,
stride=stride,
kernel_size=1,
norm='def')
else:
self.shortcut = Identity()
self.act = Act()
def __init__(self, C_in, C_out, kernel_size, stride):
super().__init__([
Act(),
Conv2d(C_in,
C_in,
kernel_size,
stride=stride,
groups=C_in,
bias=False),
Conv2d(C_in, C_in, 1, bias=False),
Norm(C_in),
Act(),
Conv2d(C_in, C_in, kernel_size, 1, groups=C_in, bias=False),
Conv2d(C_in, C_out, 1, bias=False),
Norm(C_out),
])
def __init__(self, start_channels, widening_fractor, depth, groups, radix, drop_path, num_classes=10):
super().__init__()
num_layers = [(depth - 2) // 9] * 3
strides = [1, 2, 2]
self.add_channel = widening_fractor / sum(num_layers)
self.in_channels = start_channels
self.channels = start_channels
layers = [Conv2d(3, start_channels, kernel_size=3, norm='def')]
for i, (n, s) in enumerate(zip(num_layers, strides)):
layers.append(self._make_layer(n, groups, stride=s, radix=radix, drop_path=drop_path))
layers.append(Sequential([
Norm(self.in_channels),
Act(),
]))
self.features = Sequential(layers)
assert (start_channels + widening_fractor) * Bottleneck.expansion == self.in_channels
self.final_pool = GlobalAvgPool()
self.fc = Linear(self.in_channels, num_classes)
def __init__(self,
in_channels,
channels,
stride,
erase_relu=False,
zero_init_residual=False,
dilation=None,
avd=None):
super().__init__()
out_channels = channels * self.expansion
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=stride,
norm='def',
act='def')
self.conv2 = Conv2d(out_channels,
out_channels,
kernel_size=3,
norm='def')
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(2, 2, type='avg'))
shortcut.append(
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.act = Act() if not erase_relu else Identity()
def __init__(self,
in_channels,
channels,
stride,
start_block=False,
end_block=False,
exclude_bn0=False,
conv_cls=Conv2d):
super().__init__()
out_channels = channels * self.expansion
width = getattr(self, "width", channels)
if not start_block and not exclude_bn0:
self.bn0 = Norm(in_channels)
if not start_block:
self.act0 = Act()
self.conv1 = Conv2d(in_channels, width, kernel_size=1)
self.bn1 = Norm(width)
self.act1 = Act()
self.conv2 = conv_cls(in_channels=width,
out_channels=width,
kernel_size=3,
stride=stride,
norm='def',
act='def',
start_block=start_block)
self.conv3 = Conv2d(width, out_channels, kernel_size=1)
if start_block:
self.bn3 = Norm(out_channels)
if end_block:
self.bn3 = Norm(out_channels)
self.act3 = Act()
if stride != 1 or in_channels != out_channels:
shortcut = []
if stride != 1:
shortcut.append(Pool2d(2, 2, type='avg'))
shortcut.append(
Conv2d(in_channels, out_channels, kernel_size=1, norm='def'))
self.shortcut = Sequential(shortcut)
else:
self.shortcut = Identity()
self.start_block = start_block
self.end_block = end_block
self.exclude_bn0 = exclude_bn0
def __init__(self, in_channels, out_channels, stride, dropout):
super().__init__()
self.norm1 = Norm(in_channels)
self.act1 = Act()
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=stride)
self.norm2 = Norm(out_channels)
self.act2 = Act()
self.dropout = Dropout(dropout) if dropout else Identity()
self.conv2 = Conv2d(out_channels, out_channels, kernel_size=3)
self.shortcut = Conv2d(in_channels,
out_channels,
kernel_size=1,
stride=stride)
def __init__(self,
in_channels,
out_channels,
stride,
askc_type='DirectAdd'):
super().__init__()
self.norm1 = Norm(in_channels)
self.act1 = Act()
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=stride,
bias=False)
self.norm2 = Norm(out_channels)
self.act2 = Act()
self.conv2 = Conv2d(out_channels,
out_channels,
kernel_size=3,
bias=False)
if in_channels != out_channels or stride == 2:
if stride == 2:
self.shortcut = Sequential([
Pool2d(2, 2, type='avg'),
Conv2d(in_channels,
out_channels,
kernel_size=1,
bias=False),
])
else:
self.shortcut = Conv2d(in_channels,
out_channels,
kernel_size=1,
bias=False)
else:
self.shortcut = Identity()
if askc_type == 'DirectAdd':
self.attention = DirectAddFuse()
elif askc_type == 'iAFF':
self.attention = iAFF(out_channels)
elif askc_type == 'AFF':
self.attention = AFF(out_channels)
else:
raise ValueError('Unknown askc_type')
def __init__(self, C_in, C_out):
super().__init__()
assert C_out % 2 == 0
self.act = Act()
self.slice = Slice([1, 1, 0], [-1, -1, -1])
self.conv1 = Conv2d(C_in, C_out // 2, 1, stride=2, bias=False)
self.conv2 = Conv2d(C_in, C_out // 2, 1, stride=2, bias=False)
self.norm = Norm(C_out)
self.concat = Concatenate()
def __init__(self, in_channels, out_channels, stride):
super().__init__()
self.norm1 = Norm(in_channels)
self.act1 = Act()
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=stride)
self.norm2 = Norm(out_channels)
self.act2 = Act()
self.conv2 = Conv2d(out_channels, out_channels, kernel_size=3)
if stride != 1 or in_channels != out_channels:
self.shortcut = Conv2d(in_channels,
out_channels,
kernel_size=1,
stride=stride)
self.rezero = ReZero()
def __init__(self, feat_channels, seperable_conv=True, norm='bn', act='def'):
super().__init__()
self.fusion_weights = [
self.add_weight(
f"weight{i+1}", shape=(), initializer='ones') for i in range(2)]
conv_op = SeparableConv2d if seperable_conv else Conv2d
self.conv = Sequential([
Act(act),
conv_op(feat_channels, feat_channels, 3, norm=norm)
])
def __init__(self, C_prev, C, stride=2):
super().__init__()
assert stride == 2
self.conv1 = ReLUConvBN(C_prev, C, 3, stride=stride)
self.conv2 = ReLUConvBN(C, C, 3, stride=1)
self.downsample = Sequential([
Pool2d(2, 2, type='avg'),
Conv2d(C_prev, C, 1),
])
self.act = Act()
self.stride = stride
def __init__(self,
start_channels,
alpha,
depth,
block='bottleneck',
p_shakedrop=0.5,
num_classes=10):
super().__init__()
if block == 'basic':
num_layers = [(depth - 2) // 6] * 3
block = BasicBlock
elif block == 'bottleneck':
num_layers = [(depth - 2) // 9] * 3
block = Bottleneck
else:
raise ValueError("block must be `basic` or `bottleneck`, got %s" %
block)
self.num_layers = num_layers
strides = [1, 2, 2]
add_channel = alpha / sum(num_layers)
in_channels = start_channels
self.init_block = Conv2d(3, start_channels, kernel_size=3, norm='def')
channels = start_channels
k = 1
units = []
for n, s in zip(num_layers, strides):
for i in range(n):
stride = s if i == 0 else 1
channels = channels + add_channel
units.append(
block(in_channels,
rd(channels),
stride=stride,
p_shakedrop=k / sum(num_layers) * p_shakedrop))
in_channels = rd(channels) * block.expansion
k += 1
self.units = units
self.post_activ = Sequential([
Norm(in_channels),
Act(),
])
assert (start_channels + alpha) * block.expansion == in_channels
self.final_pool = GlobalAvgPool()
self.fc = Linear(in_channels, num_classes)
def __init__(self, C, num_classes):
"""assuming input size 8x8"""
super().__init__()
self.features = Sequential([
Act(),
Pool2d(5, stride=3, padding=0, type='avg'),
Conv2d(C, 128, 1, norm='def', act='def'),
Conv2d(128, 768, 2, norm='def', act='def', padding=0),
])
self.classifier = Sequential([
GlobalAvgPool(),
Linear(768, num_classes),
])
def __init__(self, in_channels, out_channels, stride):
super().__init__()
self.norm1 = Norm(in_channels)
self.act1 = Act()
self.conv1 = Conv2d(in_channels,
out_channels,
kernel_size=3,
stride=stride)
self.norm2 = Norm(out_channels)
self.act2 = Act()
self.conv2 = Conv2d(out_channels, out_channels, kernel_size=3)
if stride != 1 or in_channels != out_channels:
self.shortcut = Conv2d(in_channels,
out_channels,
kernel_size=1,
stride=stride)
self.res_weight = self.add_weight(name='res_weight',
shape=(),
dtype=tf.float32,
trainable=True,
initializer=Constant(0.))
