def __init__(self,
full=False,
binary=True,
first_sparsity=0.8,
rest_sparsity=0.9,
hid=4096,
ind=768,
align=False):
super(FC_large, self).__init__()
self.align = align
self.pruned = False
self.hid = hid
self.ind = ind
self.full = full
self.binary = binary
if full:
self.fc1 = nn.Linear(ind, hid, bias=False)
self.fc2 = nn.Linear(hid, hid, bias=False)
self.fc3 = nn.Linear(hid, hid, bias=False)
self.fc4 = nn.Linear(hid, 10, bias=False)
elif binary:
self.fc1 = binarized_modules.BinarizeLinear(ind, hid, bias=False)
self.fc2 = binarized_modules.BinarizeLinear(hid, hid, bias=False)
self.fc3 = binarized_modules.BinarizeLinear(hid, hid, bias=False)
self.fc4 = binarized_modules.BinarizeLinear(hid, 10, bias=False)
else:
self.fc1 = binarized_modules.TernarizeLinear(first_sparsity,
ind,
hid,
bias=False,
align=align)
self.fc2 = binarized_modules.TernarizeLinear(rest_sparsity,
hid,
hid,
bias=False,
align=align)
self.fc3 = binarized_modules.TernarizeLinear(rest_sparsity,
hid,
hid,
bias=False,
align=align)
self.fc4 = binarized_modules.TernarizeLinear(rest_sparsity,
hid,
10,
bias=False,
align=align)
self.htanh1 = nn.Hardtanh()
self.bn1 = nn.BatchNorm1d(hid)
self.htanh2 = nn.Hardtanh()
self.bn2 = nn.BatchNorm1d(hid)
self.htanh3 = nn.Hardtanh()
self.bn3 = nn.BatchNorm1d(hid)
self.bn4 = nn.BatchNorm1d(10, affine=True)
self.logsoftmax = nn.LogSoftmax(dim=1)
def __init__(self):
super(BinaryLenet, self).__init__()
self.conv = nn.Sequential(
bnn.BinarizeConv2d(in_channels=1,
out_channels=20,
kernel_size=(5, 5),
padding=2),
# nn.BatchNorm2d(20),
nn.ReLU(),
# nn.Hardtanh(),
# nn.LeakyReLU(),
nn.MaxPool2d(kernel_size=(2, 2), stride=2),
bnn.BinarizeConv2d(in_channels=20,
out_channels=50,
kernel_size=(5, 5),
padding=2),
# nn.BatchNorm2d(50),
nn.ReLU(),
# nn.LeakyReLU(),
# nn.Hardtanh(),
nn.MaxPool2d(kernel_size=(2, 2), stride=2),
)
self.FC = nn.Sequential(
bnn.BinarizeLinear(7 * 7 * 50, 1024),
# nn.BatchNorm1d(1024),
nn.ReLU(),
# nn.Hardtanh(),
# nn.LeakyReLU(),
nn.Dropout(0.5),
bnn.BinarizeLinear(1024, 10),
# nn.Linear(28*28, 10),
# nn.LogSoftmax(dim=1)
)
# self.regime = {
# 0: {'optimizer': 'Adam', 'lr': 1e-3},
# 50: {'lr': 5e-6,'weight_decay': 0},
# 50: {'lr': 1e-7}
# }
self.regime = {
0: {
'optimizer': 'Adam',
'lr': 5e-3
},
101: {
'lr': 1e-3
},
142: {
'lr': 5e-4
},
184: {
'lr': 1e-4
},
220: {
'lr': 1e-5
}
}
def __init__(self,
full=False,
binary=True,
conv_thres=0.7,
fc_thres=0.9,
align=False,
pad=0):
super(CNN_large, self).__init__()
self.pruned = False
self.full = full
self.binary = binary
self.pad = pad
self.conv1 = binarized_modules.BinarizeConv2d(3,
128,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.bn1 = nn.BatchNorm2d(128)
self.htanh1 = nn.Hardtanh(inplace=True)
if full:
self.conv1 = nn.Conv2d(3,
128,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.conv2 = nn.Conv2d(128,
128,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.conv3 = nn.Conv2d(128,
256,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.conv4 = nn.Conv2d(256,
256,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.conv5 = nn.Conv2d(256,
512,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.conv6 = nn.Conv2d(512,
512,
kernel_size=3,
stride=1,
padding=1,
bias=False)
self.fc1 = nn.Linear(512 * 4 * 4, 1024, bias=False)
self.fc2 = nn.Linear(1024, 1024, bias=False)
self.fc3 = nn.Linear(1024, 10, bias=False)
elif binary:
self.conv1 = binarized_modules.BinarizeConv2d(3,
128,
kernel_size=3,
stride=1,
padding=0,
bias=False)
self.conv2 = binarized_modules.BinarizeConv2d(128,
128,
kernel_size=3,
stride=1,
padding=0,
bias=False)
self.conv3 = binarized_modules.BinarizeConv2d(128,
256,
kernel_size=3,
stride=1,
padding=0,
bias=False)
self.conv4 = binarized_modules.BinarizeConv2d(256,
256,
kernel_size=3,
stride=1,
padding=0,
bias=False)
self.conv5 = binarized_modules.BinarizeConv2d(256,
512,
kernel_size=3,
stride=1,
padding=0,
bias=False)
self.conv6 = binarized_modules.BinarizeConv2d(512,
512,
kernel_size=3,
stride=1,
padding=0,
bias=False)
self.conv7 = binarized_modules.BinarizeConv2d(512,
1024,
kernel_size=4,
padding=0,
bias=False)
self.fc1 = binarized_modules.BinarizeLinear(1024, 1024, bias=False)
self.fc2 = binarized_modules.BinarizeLinear(1024, 10, bias=False)
else:
self.conv1 = binarized_modules.BinarizeConv2d(3,
128,
kernel_size=3,
stride=1,
padding=0,
bias=False)
self.conv2 = binarized_modules.TernarizeConv2d(conv_thres,
128,
128,
kernel_size=3,
padding=0,
bias=False,
align=align)
self.conv3 = binarized_modules.TernarizeConv2d(conv_thres,
128,
256,
kernel_size=3,
padding=0,
bias=False,
align=align)
self.conv4 = binarized_modules.TernarizeConv2d(conv_thres,
256,
256,
kernel_size=3,
padding=0,
bias=False,
align=align)
self.conv5 = binarized_modules.TernarizeConv2d(conv_thres,
256,
512,
kernel_size=3,
padding=0,
bias=False,
align=align)
self.conv6 = binarized_modules.TernarizeConv2d(conv_thres,
512,
512,
kernel_size=3,
padding=0,
bias=False,
align=align)
self.conv7 = binarized_modules.TernarizeConv2d(fc_thres,
512,
1024,
kernel_size=4,
padding=0,
bias=False,
align=align)
self.fc1 = binarized_modules.TernarizeLinear(fc_thres,
1024,
1024,
bias=False,
align=align)
self.fc2 = binarized_modules.TernarizeLinear(0.49,
1024,
10,
bias=False)
self.pool2 = nn.MaxPool2d(kernel_size=2, stride=2)
self.bn2 = nn.BatchNorm2d(128)
self.htanh2 = nn.Hardtanh(inplace=True)
self.bn3 = nn.BatchNorm2d(256)
self.htanh3 = nn.Hardtanh(inplace=True)
self.pool4 = nn.MaxPool2d(kernel_size=2, stride=2)
self.bn4 = nn.BatchNorm2d(256)
self.htanh4 = nn.Hardtanh(inplace=True)
self.bn5 = nn.BatchNorm2d(512)
self.htanh5 = nn.Hardtanh(inplace=True)
self.pool6 = nn.MaxPool2d(kernel_size=2, stride=2)
self.bn6 = nn.BatchNorm2d(512)
self.htanh6 = nn.Hardtanh(inplace=True)
self.bnfc1 = nn.BatchNorm1d(1024)
self.htanhfc1 = nn.Hardtanh(inplace=True)
self.bnfc2 = nn.BatchNorm1d(1024)
self.htanhfc2 = nn.Hardtanh(inplace=True)
self.bnfc3 = nn.BatchNorm1d(10, affine=True)
self.logsoftmax = nn.LogSoftmax(dim=1)
self.regime = {
0: {
'optimizer': 'Adam',
'betas': (0.9, 0.999),
'lr': 5e-3
},
40: {
'lr': 1e-3
},
80: {
'lr': 5e-4
},
100: {
'lr': 1e-4
},
120: {
'lr': 5e-5
},
140: {
'lr': 1e-5
}
}