def __init__(self, margin, device):
super(MNISTFIG2Net, self).__init__()
self.margin = margin
self.device = device
self.conv_1 = nn.Sequential(nn.Conv2d(1, 32, 5, padding=2), nn.PReLU(),
nn.BatchNorm2d(32),
nn.Conv2d(32, 32, 5, padding=2),
nn.PReLU(), nn.BatchNorm2d(32),
nn.MaxPool2d(2, 2))
self.conv_2 = nn.Sequential(nn.Conv2d(32, 64, 5,
padding=2), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 5, padding=2),
nn.PReLU(), nn.BatchNorm2d(64),
nn.MaxPool2d(2, 2))
self.conv_3 = nn.Sequential(nn.Conv2d(64, 128, 5, padding=2),
nn.PReLU(), nn.BatchNorm2d(128),
nn.Conv2d(128, 128, 5, padding=2),
nn.PReLU(), nn.BatchNorm2d(128),
nn.MaxPool2d(2, 2))
self.fc = nn.Sequential(nn.Linear(1152, 2), nn.BatchNorm1d(2))
self.lsoftmax_linear = LSoftmaxLinear(input_features=2,
output_features=10,
margin=margin,
device=self.device)
self.reset_parameters()
class MNISTNet(nn.Module):
def __init__(self, margin, device):
super(MNISTNet, self).__init__()
self.margin = margin
self.device = device
self.conv_0 = nn.Sequential(nn.BatchNorm2d(1), nn.Conv2d(1, 64, 3),
nn.PReLU(), nn.BatchNorm2d(64))
self.conv_1 = nn.Sequential(nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3, padding=1),
nn.PReLU(), nn.BatchNorm2d(64),
nn.MaxPool2d(2, 2))
self.conv_2 = nn.Sequential(nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3, padding=1),
nn.PReLU(), nn.BatchNorm2d(64),
nn.MaxPool2d(2, 2))
self.conv_3 = nn.Sequential(nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3, padding=1),
nn.PReLU(), nn.BatchNorm2d(64),
nn.MaxPool2d(2, 2))
self.fc = nn.Sequential(nn.Linear(576, 256), nn.BatchNorm1d(256))
self.lsoftmax_linear = LSoftmaxLinear(input_features=256,
output_features=10,
margin=margin,
device=self.device)
self.reset_parameters()
def reset_parameters(self):
self.lsoftmax_linear.reset_parameters()
def forward(self, x, target=None):
x = self.conv_0(x)
x = self.conv_1(x)
x = self.conv_2(x)
x = self.conv_3(x)
x = x.view(-1, 576)
x = self.fc(x)
logit = self.lsoftmax_linear(input=x, target=target)
return logit
def test_trivial(self):
m = LSoftmaxLinear(input_dim=2, output_dim=4, margin=1)
x = Variable(torch.arange(6).view(3, 2))
target = Variable(torch.LongTensor([3, 0, 1]))
m.eval()
eval_out = m.forward(input=x).data.tolist()
m.train()
train_out = m.forward(input=x, target=target).data.tolist()
np.testing.assert_array_almost_equal(eval_out, train_out)
def __init__(self, margin):
super().__init__()
self.margin = margin
cnn_layers = OrderedDict()
cnn_layers['pre_bn'] = nn.BatchNorm2d(1)
cnn_layers['conv0_0'] = nn.Conv2d(in_channels=1, out_channels=64,
kernel_size=(3, 3), padding=1)
cnn_layers['prelu0_0'] = nn.PReLU(64)
cnn_layers['bn0_0'] = nn.BatchNorm2d(64)
# conv1.x
for x in range(3):
cnn_layers[f'conv1_{x}'] = nn.Conv2d(
in_channels=64, out_channels=64, kernel_size=(3, 3),
padding=1)
cnn_layers[f'prelu1_{x}'] = nn.PReLU(64)
cnn_layers[f'bn1_{x}'] = nn.BatchNorm2d(64)
cnn_layers['pool1'] = nn.MaxPool2d(kernel_size=(2, 2), stride=2)
# conv2.x
for x in range(4):
cnn_layers[f'conv2_{x}'] = nn.Conv2d(
in_channels=64, out_channels=64, kernel_size=(3, 3),
padding=1)
cnn_layers[f'prelu2_{x}'] = nn.PReLU(64)
cnn_layers[f'bn2_{x}'] = nn.BatchNorm2d(64)
cnn_layers['pool2'] = nn.MaxPool2d(kernel_size=(2, 2), stride=2)
# conv3.x
for x in range(4):
cnn_layers[f'conv3_{x}'] = nn.Conv2d(
in_channels=64, out_channels=64, kernel_size=(3, 3),
padding=1)
cnn_layers[f'prelu3_{x}'] = nn.PReLU(64)
cnn_layers[f'bn3_{x}'] = nn.BatchNorm2d(64)
cnn_layers['pool3'] = nn.MaxPool2d(kernel_size=(2, 2), stride=2)
self.net = nn.Sequential(cnn_layers)
self.fc = nn.Sequential(OrderedDict([
('fc0', nn.Linear(in_features=576, out_features=256)),
# ('fc1', nn.Linear(in_features=256, out_features=10))
('fc0_bn', nn.BatchNorm1d(256))
]))
self.lsoftmax_linear = LSoftmaxLinear(
input_dim=256, output_dim=10, margin=margin)
self.reset_parameters()
def __init__(self, margin, device):
super(MNISTNet, self).__init__()
self.margin = margin
self.device = device
self.conv_0 = nn.Sequential(nn.BatchNorm2d(1), nn.Conv2d(1, 64, 3),
nn.PReLU(), nn.BatchNorm2d(64))
self.conv_1 = nn.Sequential(nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3, padding=1),
nn.PReLU(), nn.BatchNorm2d(64),
nn.MaxPool2d(2, 2))
self.conv_2 = nn.Sequential(nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3, padding=1),
nn.PReLU(), nn.BatchNorm2d(64),
nn.MaxPool2d(2, 2))
self.conv_3 = nn.Sequential(nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3,
padding=1), nn.PReLU(),
nn.BatchNorm2d(64),
nn.Conv2d(64, 64, 3, padding=1),
nn.PReLU(), nn.BatchNorm2d(64),
nn.MaxPool2d(2, 2))
self.fc = nn.Sequential(nn.Linear(576, 256), nn.BatchNorm1d(256))
self.lsoftmax_linear = LSoftmaxLinear(input_features=256,
output_features=10,
margin=margin,
device=self.device)
self.reset_parameters()
def __init__(self,
block,
layers,
device,
num_classes=2,
zero_init_residual=False,
groups=1,
width_per_group=64,
replace_stride_with_dilation=None,
norm_layer=None):
super(ResNetLSoftmax,
self).__init__(block, layers, num_classes, zero_init_residual,
groups, width_per_group,
replace_stride_with_dilation, norm_layer)
self.fc = LSoftmaxLinear(input_features=512,
output_features=2,
margin=2,
device=device)
self.fc.reset_parameters()
class ResNetLSoftmax(ResNet):
def __init__(self,
block,
layers,
device,
num_classes=2,
zero_init_residual=False,
groups=1,
width_per_group=64,
replace_stride_with_dilation=None,
norm_layer=None):
super(ResNetLSoftmax,
self).__init__(block, layers, num_classes, zero_init_residual,
groups, width_per_group,
replace_stride_with_dilation, norm_layer)
self.fc = LSoftmaxLinear(input_features=512,
output_features=2,
margin=2,
device=device)
self.fc.reset_parameters()
def forward(self, x, target=None):
x = self.conv1(x)
x = self.bn1(x)
x = self.relu(x)
x = self.maxpool(x)
x = self.layer1(x)
x = self.layer2(x)
x = self.layer3(x)
x = self.layer4(x)
x = self.avgpool(x)
x = x.reshape(x.size(0), -1)
x = self.fc(input=x, target=target)
return x
def test_even(self):
m = LSoftmaxLinear(input_dim=2, output_dim=4, margin=4)
m.weight.data.copy_(torch.arange(-4, 4).view(2, 4))
x = Variable(torch.arange(6).view(3, 2))
target = Variable(torch.LongTensor([3, 0, 1]))
m.eval()
eval_out = m.forward(input=x).data.tolist()
eval_gold = [[0, 1, 2, 3], [-8, -3, 2, 7], [-16, -7, 2, 11]]
np.testing.assert_array_almost_equal(eval_out, eval_gold, decimal=5)
m.train()
train_out = m.forward(input=x, target=target).data.tolist()
train_gold = [[0, 1, 2, 0.88543774484714499],
[-67.844100922931872, -3, 2, 7],
[-16, -77.791173935544478, 2, 11]]
np.testing.assert_array_almost_equal(train_out, train_gold, decimal=5)
def test_odd(self):
m = LSoftmaxLinear(input_dim=2, output_dim=4, margin=3)
m.weight.data.copy_(torch.arange(-4, 4).view(2, 4))
x = Variable(torch.arange(6).view(3, 2))
target = Variable(torch.LongTensor([3, 0, 1]))
m.eval()
eval_out = m.forward(input=x).data.tolist()
eval_gold = [[0, 1, 2, 3], [-8, -3, 2, 7], [-16, -7, 2, 11]]
np.testing.assert_array_almost_equal(eval_out, eval_gold, decimal=5)
m.train()
train_out = m.forward(input=x, target=target).data.tolist()
train_gold = [[0, 1, 2, 1.7999999999999996],
[-43.53497425357768, -3, 2, 7],
[-16, -58.150571999218542, 2, 11]]
np.testing.assert_array_almost_equal(train_out, train_gold, decimal=5)
def __init__(self, lsoftmax=None, margin=None):
super(Deepyeast, self).__init__()
self.block1conv1 = nn.Sequential(nn.Conv2d(2, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True))
self.block1conv2 = nn.Sequential(nn.Conv2d(64, 64, 3, padding=1),
nn.BatchNorm2d(64),
nn.ReLU(inplace=True))
self.maxpool1 = nn.MaxPool2d(kernel_size=2, stride=2)
self.block2conv1 = nn.Sequential(nn.Conv2d(64, 128, 3, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(inplace=True))
self.block2conv2 = nn.Sequential(nn.Conv2d(128, 128, 3, padding=1),
nn.BatchNorm2d(128),
nn.ReLU(inplace=True))
self.maxpool2 = nn.MaxPool2d(kernel_size=2, stride=2)
self.block3conv1 = nn.Sequential(nn.Conv2d(128, 256, 3, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True))
self.block3conv2 = nn.Sequential(nn.Conv2d(256, 256, 3, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True))
self.block3conv3 = nn.Sequential(nn.Conv2d(256, 256, 3, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True))
self.block3conv4 = nn.Sequential(nn.Conv2d(256, 256, 3, padding=1),
nn.BatchNorm2d(256),
nn.ReLU(inplace=True))
self.maxpool3 = nn.MaxPool2d(kernel_size=2, stride=2)
self.fc1 = nn.Sequential(nn.Linear(16384, 512), nn.BatchNorm1d(512),
nn.ReLU(inplace=True), nn.Dropout())
self.fc2 = nn.Sequential(nn.Linear(512, 512), nn.BatchNorm1d(512),
nn.ReLU(inplace=True), nn.Dropout())
self.fc3 = nn.Linear(512, 12)
self.dropout2 = nn.Dropout()
self.out = nn.LogSoftmax(dim=1)
self.lsoftmax_linear = LSoftmaxLinear(input_features=512,
output_features=12,
margin=margin)
self.iflsoftmax = lsoftmax
def __init__(self, growth_rate=4, block_config=(6, 6, 6), compression=0.5,
num_init_features=24, bn_size=4, drop_rate=0, avgpool_size=8,
num_classes=10,margin=2):
super(DenseNet, self).__init__()
# The first Convolution layer
self.features = nn.Sequential(OrderedDict([
('conv0', nn.Conv2d(1, num_init_features,
kernel_size=3, stride=1, padding=1, bias=False)),
('norm0', nn.BatchNorm2d(num_init_features)),
('relu0', nn.ReLU(inplace=True)),
]))
# Did not add the pooling layer to preserve dimension
# The number of layers in each Densnet is adjustable
num_features = num_init_features
for i, num_layers in enumerate(block_config):
Dense_block = _DenseBlock(num_layers=num_layers, num_input_features=num_features,
bn_size=bn_size, growth_rate=growth_rate, drop_rate=drop_rate)
# Add name to the Denseblock
self.features.add_module('denseblock%d' % (i + 1), Dense_block)
# Increase the number of features by the growth rate times the number
# of layers in each Denseblock
num_features += num_layers * growth_rate
# check whether the current block is the last block
# Add a transition layer to all Denseblocks except the last
if i != len(block_config):
# Reduce the number of output features in the transition layer
nOutChannels = int(math.floor(num_features*compression))
trans = _Transition(num_input_features=num_features,
num_output_features=nOutChannels)
self.features.add_module('transition%d' % (i + 1), trans)
# change the number of features for the next Dense block
num_features = nOutChannels
# Final batch norm
self.features.add_module('last_norm%d' % (i+1), nn.BatchNorm2d(num_features))
# Linear layer
self.classifier = nn.Linear(num_features, num_features)
self.lsoftmax_linear = LSoftmaxLinear(input_dim=num_features,output_dim = num_classes,margin = margin)
class MNISTModel(nn.Module):
def __init__(self, margin):
super().__init__()
self.margin = margin
cnn_layers = OrderedDict()
cnn_layers['pre_bn'] = nn.BatchNorm2d(1)
cnn_layers['conv0_0'] = nn.Conv2d(in_channels=1, out_channels=64,
kernel_size=(3, 3), padding=1)
cnn_layers['prelu0_0'] = nn.PReLU(64)
cnn_layers['bn0_0'] = nn.BatchNorm2d(64)
# conv1.x
for x in range(3):
cnn_layers[f'conv1_{x}'] = nn.Conv2d(
in_channels=64, out_channels=64, kernel_size=(3, 3),
padding=1)
cnn_layers[f'prelu1_{x}'] = nn.PReLU(64)
cnn_layers[f'bn1_{x}'] = nn.BatchNorm2d(64)
cnn_layers['pool1'] = nn.MaxPool2d(kernel_size=(2, 2), stride=2)
# conv2.x
for x in range(4):
cnn_layers[f'conv2_{x}'] = nn.Conv2d(
in_channels=64, out_channels=64, kernel_size=(3, 3),
padding=1)
cnn_layers[f'prelu2_{x}'] = nn.PReLU(64)
cnn_layers[f'bn2_{x}'] = nn.BatchNorm2d(64)
cnn_layers['pool2'] = nn.MaxPool2d(kernel_size=(2, 2), stride=2)
# conv3.x
for x in range(4):
cnn_layers[f'conv3_{x}'] = nn.Conv2d(
in_channels=64, out_channels=64, kernel_size=(3, 3),
padding=1)
cnn_layers[f'prelu3_{x}'] = nn.PReLU(64)
cnn_layers[f'bn3_{x}'] = nn.BatchNorm2d(64)
cnn_layers['pool3'] = nn.MaxPool2d(kernel_size=(2, 2), stride=2)
self.net = nn.Sequential(cnn_layers)
self.fc = nn.Sequential(OrderedDict([
('fc0', nn.Linear(in_features=576, out_features=256)),
# ('fc1', nn.Linear(in_features=256, out_features=10))
('fc0_bn', nn.BatchNorm1d(256))
]))
self.lsoftmax_linear = LSoftmaxLinear(
input_dim=256, output_dim=10, margin=margin)
self.reset_parameters()
def reset_parameters(self):
def init_kaiming(layer):
init.kaiming_normal(layer.weight.data)
init.constant(layer.bias.data, val=0)
init_kaiming(self.net.conv0_0)
for x in range(3):
init_kaiming(getattr(self.net, f'conv1_{x}'))
for x in range(4):
init_kaiming(getattr(self.net, f'conv2_{x}'))
for x in range(4):
init_kaiming(getattr(self.net, f'conv3_{x}'))
init_kaiming(self.fc.fc0)
self.lsoftmax_linear.reset_parameters()
# init_kaiming(self.fc.fc1)
def forward(self, input, target=None):
"""
Args:
input: A variable of size (N, 1, 28, 28).
target: A long variable of size (N,).
Returns:
logit: A variable of size (N, 10).
"""
conv_output = self.net(input)
batch_size = conv_output.size(0)
fc_input = conv_output.view(batch_size, -1)
fc_output = self.fc(fc_input)
logit = self.lsoftmax_linear(input=fc_output, target=target)
return logit