def __init__(self):
super(LeKervNet, self).__init__()
self.kerv1 = nn.Sequential(
nn.Kerv2d(
in_channels=1, # input height
out_channels=6, # n_filters
kernel_size=5, # filter size
stride=1, # filter movement/step
padding=2, # if want same width and length of this image after con2d, padding=(kernel_size-1)/2 if stride=1
kernel_type = args.kernel_type,
learnable_kernel = args.learnable_kernel,
), # input shape (1, 28, 28)
nn.ReLU(), # activation
nn.MaxPool2d(2), # output shape (6, 14, 14)
)
self.kerv2 = nn.Sequential( # input shape (6, 14, 14)
nn.Kerv2d(6,16,5,1,0, # output shape (16, 10, 10)
mapping='translation',
kernel_type='linear'
),
nn.ReLU(), # activation
nn.MaxPool2d(2), # output shape (16, 5, 5)
)
self.fc1 = nn.Linear(16 * 5 * 5, 120) # fully connected layer, output 10 classes
self.fc2 = nn.Linear(120, 84)
self.fc3 = nn.Linear(84, 10)
def __init__(self,
num_input_features,
growth_rate,
bn_size,
drop_rate,
memory_efficient=False):
super(_DenseLayer, self).__init__()
self.add_module('norm1', nn.BatchNorm2d(num_input_features)),
self.add_module('relu1', nn.ReLU(inplace=True)),
self.add_module(
'Kerv1',
nn.Kerv2d(num_input_features,
bn_size * growth_rate,
kernel_size=1,
stride=1,
bias=False)),
self.add_module('norm2', nn.BatchNorm2d(bn_size * growth_rate)),
self.add_module('relu2', nn.ReLU(inplace=True)),
self.add_module(
'Kerv2',
nn.Kerv2d(bn_size * growth_rate,
growth_rate,
kernel_size=3,
stride=1,
padding=1,
bias=False)),
self.drop_rate = float(drop_rate)
self.memory_efficient = memory_efficient
def __init__(self, in_planes, planes, stride=1):
super(BasicBlock, self).__init__()
self.kerv1 = nn.Kerv2d(in_planes, planes, kernel_size=3, stride=stride, padding=1, bias=False)
self.bn1 = nn.BatchNorm2d(planes)
self.kerv2 = nn.Kerv2d(planes, planes, kernel_size=3, stride=1, padding=1, bias=False)
self.bn2 = nn.BatchNorm2d(planes)
self.shortcut = nn.Sequential()
if stride != 1 or in_planes != self.expansion*planes:
self.shortcut = nn.Sequential(
nn.Kerv2d(in_planes, self.expansion*planes, kernel_size=1, stride=stride, bias=False),
nn.BatchNorm2d(self.expansion*planes)
)
def __init__(self, block, layers, num_classes=1000):
self.inplanes = 64
super(ResNet, self).__init__()
self.conv1 = nn.Kerv2d(3,
64,
kernel_size=7,
stride=2,
padding=3,
bias=False,
kernel_type='polynomial',
learnable_kernel=False,
kernel_regularizer=False)
self.bn1 = nn.BatchNorm2d(64)
self.relu = nn.ReLU(inplace=True)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(block, 64, layers[0])
self.layer2 = self._make_layer(block, 128, layers[1], stride=2)
self.layer3 = self._make_layer(block, 256, layers[2], stride=2)
self.layer4 = self._make_layer(block, 512, layers[3], stride=2)
self.avgpool = nn.AvgPool2d(7)
self.fc = nn.Linear(512 * block.expansion, num_classes)
for m in self.modules():
if isinstance(m, nn.Conv2d):
if m.kernel_size == 7: # for kervolution initialization
n = m.kernel_size * m.kernel_size * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
else:
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def __init__(self, num_classes=10):
super(KGoogLeNet, self).__init__()
self.pre_layers = nn.Sequential(
nn.Kerv2d(
3,
192,
kernel_size=3,
padding=1,
kernel_type='polynomial',
learnable_kernel=True,
),
nn.BatchNorm2d(192),
nn.ReLU(True),
)
self.a3 = Inception(192, 64, 96, 128, 16, 32, 32)
self.b3 = Inception(256, 128, 128, 192, 32, 96, 64)
self.maxpool = nn.MaxPool2d(3, stride=2, padding=1)
self.a4 = Inception(480, 192, 96, 208, 16, 48, 64)
self.b4 = Inception(512, 160, 112, 224, 24, 64, 64)
self.c4 = Inception(512, 128, 128, 256, 24, 64, 64)
self.d4 = Inception(512, 112, 144, 288, 32, 64, 64)
self.e4 = Inception(528, 256, 160, 320, 32, 128, 128)
self.a5 = Inception(832, 256, 160, 320, 32, 128, 128)
self.b5 = Inception(832, 384, 192, 384, 48, 128, 128)
self.avgpool = nn.AvgPool2d(8, stride=1)
self.linear = nn.Linear(1024, num_classes)
def __init__(self):
super(KervNet, self).__init__()
self.features = nn.Sequential(
nn.Kerv2d(1, 32, kernel_size=3, stride=1, padding=1, kernel_type=args.kernel_type, learnable_kernel=args.learnable_kernel),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.Kerv2d(32, 32, kernel_size=3, stride=1, padding=1),
nn.BatchNorm2d(32),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2),
nn.Kerv2d(32, 64, kernel_size=3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.Kerv2d(64, 64, kernel_size=3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True),
nn.MaxPool2d(kernel_size=2, stride=2)
)
self.classifier = nn.Sequential(
nn.Dropout(p = 0.5),
nn.Linear(64 * 7 * 7, 512),
nn.BatchNorm1d(512),
nn.ReLU(inplace=True),
nn.Dropout(p = 0.5),
nn.Linear(512, 512),
nn.BatchNorm1d(512),
nn.ReLU(inplace=True),
nn.Dropout(p = 0.5),
nn.Linear(512, 10),
)
for m in self.features.children():
if isinstance(m, nn.Conv2d):
n = m.kernel_size[0] * m.kernel_size[1] * m.out_channels
m.weight.data.normal_(0, math.sqrt(2. / n))
elif isinstance(m, nn.BatchNorm2d):
m.weight.data.fill_(1)
m.bias.data.zero_()
for m in self.classifier.children():
if isinstance(m, nn.Linear):
nn.init.xavier_uniform(m.weight)
elif isinstance(m, nn.BatchNorm1d):
m.weight.data.fill_(1)
m.bias.data.zero_()
def kerv3x3(in_planes, out_planes, stride=1):
"3x3 convolution with padding"
return nn.Kerv2d(in_planes,
out_planes,
kernel_size=3,
stride=stride,
padding=1,
bias=False)
def __init__(self,
growth_rate=32,
block_config=(6, 12, 24, 16),
num_init_features=64,
bn_size=4,
drop_rate=0,
num_classes=1000,
memory_efficient=False):
super(DenseNet, self).__init__()
# First Kervolution
self.features = nn.Sequential(
OrderedDict([
('Kerv0',
nn.Kerv2d(3,
num_init_features,
kernel_size=7,
stride=2,
padding=3,
bias=False)),
('norm0', nn.BatchNorm2d(num_init_features)),
('relu0', nn.ReLU(inplace=True)),
('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1)),
]))
# Each denseblock
num_features = num_init_features
for i, num_layers in enumerate(block_config):
block = _DenseBlock(num_layers=num_layers,
num_input_features=num_features,
bn_size=bn_size,
growth_rate=growth_rate,
drop_rate=drop_rate,
memory_efficient=memory_efficient)
self.features.add_module('denseblock%d' % (i + 1), block)
num_features = num_features + num_layers * growth_rate
if i != len(block_config) - 1:
trans = _Transition(num_input_features=num_features,
num_output_features=num_features // 2)
self.features.add_module('transition%d' % (i + 1), trans)
num_features = num_features // 2
# Final batch norm
self.features.add_module('norm5', nn.BatchNorm2d(num_features))
# Linear layer
self.classifier = nn.Linear(num_features, num_classes)
# Official init from torch repo.
for m in self.modules():
if isinstance(m, nn.Kerv2d):
nn.init.kaiming_normal_(m.weight)
elif isinstance(m, nn.BatchNorm2d):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
elif isinstance(m, nn.Linear):
nn.init.constant_(m.bias, 0)
def __init__(self, din):
super(_RPN, self).__init__()
self.din = din # get depth of input feature map, e.g., 512
self.anchor_scales = cfg.ANCHOR_SCALES
self.anchor_ratios = cfg.ANCHOR_RATIOS
self.feat_stride = cfg.FEAT_STRIDE[0]
# define the convrelu layers processing input feature map
self.RPN_Conv = nn.Kerv2d(self.din,
512,
3,
1,
1,
mapping='translation',
kernel_type='polynomial',
learnable_kernel=True,
kernel_regularizer=False,
bias=True)
# self.RPN_Conv = nn.Conv2d(self.din, 512, 3, 1, 1, bias=True)
# define bg/fg classifcation score layer
self.nc_score_out = len(self.anchor_scales) * len(
self.anchor_ratios) * 2 # 2(bg/fg) * 9 (anchors)
self.RPN_cls_score = nn.Kerv2d(512, self.nc_score_out, 1, 1, 0)
# define anchor box offset prediction layer
self.nc_bbox_out = len(self.anchor_scales) * len(
self.anchor_ratios) * 4 # 4(coords) * 9 (anchors)
self.RPN_bbox_pred = nn.Kerv2d(512, self.nc_bbox_out, 1, 1, 0)
# define proposal layer
self.RPN_proposal = _ProposalLayer(self.feat_stride,
self.anchor_scales,
self.anchor_ratios)
# define anchor target layer
self.RPN_anchor_target = _AnchorTargetLayer(self.feat_stride,
self.anchor_scales,
self.anchor_ratios)
self.rpn_loss_cls = 0
self.rpn_loss_box = 0
def __init__(self, num_input_features, num_output_features):
super(_Transition, self).__init__()
self.add_module('norm', nn.BatchNorm2d(num_input_features))
self.add_module('relu', nn.ReLU(inplace=True))
self.add_module(
'Kerv',
nn.Kerv2d(num_input_features,
num_output_features,
kernel_size=1,
stride=1,
bias=False))
self.add_module('pool', nn.AvgPool2d(kernel_size=2, stride=2))
def __init__(self, block, num_blocks, num_classes=10):
super(KResNet, self).__init__()
self.in_planes = 64
# self.kerv1 = nn.Kerv2d(3, 64, 7, 1, 2,
self.kerv1 = nn.Kerv2d(3, 64, 3, 1,
kernel_type='polynomial',
learnable_kernel=True,
kernel_regularizer=False)
self.bn1 = nn.BatchNorm2d(64)
self.layer1 = self._make_layer(block, 64, num_blocks[0], stride=1)
self.layer2 = self._make_layer(block, 128, num_blocks[1], stride=2)
self.layer3 = self._make_layer(block, 256, num_blocks[2], stride=2)
self.layer4 = self._make_layer(block, 512, num_blocks[3], stride=2)
self.linear = nn.Linear(512*block.expansion, num_classes)
def __init__(self, in_planes, n1x1, n3x3red, n3x3, n5x5red, n5x5,
pool_planes):
super(Inception, self).__init__()
# 1x1 conv branch
self.b1 = nn.Sequential(
nn.Kerv2d(in_planes, n1x1, kernel_size=1),
nn.BatchNorm2d(n1x1),
nn.ReLU(True),
)
# 1x1 conv -> 3x3 conv branch
self.b2 = nn.Sequential(
nn.Kerv2d(in_planes, n3x3red, kernel_size=1),
nn.BatchNorm2d(n3x3red),
nn.ReLU(True),
nn.Kerv2d(n3x3red, n3x3, kernel_size=3, padding=1),
nn.BatchNorm2d(n3x3),
nn.ReLU(True),
)
# 1x1 conv -> 5x5 conv branch
self.b3 = nn.Sequential(
nn.Kerv2d(in_planes, n5x5red, kernel_size=1),
nn.BatchNorm2d(n5x5red),
nn.ReLU(True),
nn.Kerv2d(n5x5red, n5x5, kernel_size=3, padding=1),
nn.BatchNorm2d(n5x5),
nn.ReLU(True),
nn.Kerv2d(n5x5, n5x5, kernel_size=3, padding=1),
nn.BatchNorm2d(n5x5),
nn.ReLU(True),
)
# 3x3 pool -> 1x1 conv branch
self.b4 = nn.Sequential(
nn.MaxPool2d(3, stride=1, padding=1),
nn.Kerv2d(in_planes, pool_planes, kernel_size=1),
nn.BatchNorm2d(pool_planes),
nn.ReLU(True),
)
outputs = 1 / (1 + (weight_norm + input_norm - 2 * y) /
(self.sigma**2))
else:
return NotImplementedError()
if self.kernel_regularizer:
outputs = outputs + self.alpha * self.weights.abs().mean()
return outputs
def parameters(self):
if self.learnable_kernel:
print(
'alpha: %.3f, power: %.2f, balance: %.2f, sigma: %.2f, gamma: %.2f'
% (self.alpha.data[0], self.power, self.balance.data[0],
self.sigma.data[0], self.gamma.data[0]))
nn.Kerv2d = Kerv2d
if __name__ == '__main__':
kerv = nn.Kerv2d(
in_channels=2, # input height
out_channels=3, # n_filters
kernel_size=3, # filter size
stride=1, # filter movement/step
padding=1, # input padding
mapping='random', # mapping
kernel_type='polynomial', # kernel type
learnable_kernel=True) # enable learning parameters
