def __init__(self, opt, small_inputs=True, efficient=True):
super(DenseNet, self).__init__()
width = opt.width # opt.width -> growth_rate
if (opt.depth - 4) % 3: raise Exception('Depth should be of the form 3n+4')
n = (opt.depth - 4) // 6
block_config = (n, n, n)
compression = 0.5
assert 0 < compression <= 1, 'compression of densenet should be between 0 and 1'
num_init_features = opt.width*2
bn_size = 4
drop_rate = 0
num_classes = opt.num_classes
self.avgpool_size = 8 if small_inputs else 7
# First convolution
if small_inputs:
self.features = nn.Sequential(OrderedDict([
('conv0', nn.Conv2d(3, num_init_features, kernel_size=3, stride=1, padding=1, bias=False)),
]))
if opt.dataset == 'TinyImagenet':
self.features.add_module('norm0', nn.BatchNorm2d(num_init_features))
self.features.add_module('relu0', nn.ReLU(inplace=True))
self.features.add_module('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1,
ceil_mode=False))
else:
self.features = nn.Sequential(OrderedDict([
('conv0', nn.Conv2d(3, num_init_features, kernel_size=7, stride=2, padding=3, bias=False)),
]))
self.features.add_module('norm0', nn.BatchNorm2d(num_init_features))
self.features.add_module('relu0', nn.ReLU(inplace=True))
self.features.add_module('pool0', nn.MaxPool2d(kernel_size=3, stride=2, padding=1,
ceil_mode=False))
# Each denseblock
num_features = num_init_features
for i, num_layers in enumerate(block_config):
block = _DenseBlock(
num_layers=num_layers,
num_input_features=num_features,
bn_size=bn_size,
growth_rate=width,
drop_rate=drop_rate,
efficient=efficient,
)
self.features.add_module('denseblock%d' % (i + 1), block)
num_features = num_features + num_layers * width
if i != len(block_config) - 1:
trans = _Transition(num_input_features=num_features,
num_output_features=int(num_features * compression))
self.features.add_module('transition%d' % (i + 1), trans)
num_features = int(num_features * compression)
# Final batch norm
self.features.add_module('norm_final', nn.BatchNorm2d(num_features))
# Linear layer
self.dim_out = num_features
self.final = FinalBlock(opt=opt, in_channels=num_features)
def __init__(self, opt):
super(MLP, self).__init__()
self.input = FCBlock(opt=opt, in_channels=784, out_channels=opt.width)
self.hidden1 = FCBlock(opt=opt,
in_channels=opt.width,
out_channels=opt.width)
self.dim_out = opt.width
self.final = FinalBlock(opt=opt, in_channels=opt.width)
def __init__(self, opt):
super(ELM_cifar, self).__init__()
self.orth_mat = torch.nn.init.orthogonal_(
torch.empty(32 * 32 * 3, opt.emb)).to(torch.float16).to('cuda')
self.input = FCBlock(opt=opt,
in_channels=opt.emb,
out_channels=opt.width)
self.hidden1 = FCBlock(opt=opt,
in_channels=opt.width,
out_channels=opt.width)
self.final = FinalBlock(opt=opt, in_channels=opt.width)
dobj.gen_cl_mapping()
# Scenario #1: First pretraining on n classes, then do continual learning
if opt.num_pretrain_classes > 0:
opt.num_classes = opt.num_pretrain_classes
if opt.inp_size == 28: model = getattr(mnist, opt.model)(opt)
if opt.inp_size == 32 or opt.inp_size == 64: model = getattr(cifar, opt.model)(opt)
if opt.inp_size ==224: model = getattr(imagenet, opt.model)(opt)
if not os.path.isfile(opt.log_dir+opt.old_exp_name+'/pretrained_model.pth.tar'):
console_logger.debug("==> Starting pre-training..")
_, model = experiment(opt=opt, class_mask=torch.ones(opt.num_classes,opt.num_classes).cuda(), train_loader=dobj.pretrain_loader, \
test_loader=dobj.pretest_loader, model=model, logger=console_logger, num_passes=opt.num_pretrain_passes)
save_model(opt, model) # Saves the pretrained model. Subsequent CL experiments directly load the pretrained model.
else:
model = load_model(opt, model, console_logger)
# Reset the final block to new set of classes
opt.num_classes = opt.num_classes_per_task*opt.num_tasks
model.final = FinalBlock(opt, model.dim_out)
# Scenario #2: Directly do continual learning from scratch
else:
opt.num_classes = opt.num_classes_per_task*opt.num_tasks
if opt.inp_size == 28: model = getattr(mnist, opt.model)(opt)
if opt.inp_size == 32 or opt.inp_size == 64: model = getattr(cifar, opt.model)(opt)
if opt.inp_size ==224: model = getattr(imagenet, opt.model)(opt)
console_logger.debug("==> Starting Continual Learning Training..")
best_acc1, model = experiment(opt=opt, class_mask=dobj.class_mask, train_loader=dobj.cltrain_loader, test_loader=dobj.cltest_loader, \
model=model, logger=console_logger, num_passes=opt.num_passes)
console_logger.debug("==> Completed!")
def __init__(self,
opt,
block,
layers,
zero_init_residual=False,
groups=1,
width_per_group=64):
super(ResNetBase, self).__init__()
self.inplanes = 64
self.opt = opt
self.groups = groups
self.base_width = width_per_group
self.conv1block = InitialBlock(opt,
self.inplanes,
kernel_size=7,
stride=2,
padding=3,
bias=False)
self.maxpool = nn.MaxPool2d(kernel_size=3, stride=2, padding=1)
self.layer1 = self._make_layer(opt=opt,
block=block,
planes=64,
blocks=layers[0])
self.layer2 = self._make_layer(opt=opt,
block=block,
planes=128,
blocks=layers[1],
stride=2)
self.layer3 = self._make_layer(opt=opt,
block=block,
planes=256,
blocks=layers[2],
stride=2)
self.layer4 = self._make_layer(opt=opt,
block=block,
planes=512,
blocks=layers[3],
stride=2)
self.avgpool = nn.AdaptiveAvgPool2d((1, 1))
self.dim_out = in_channels = 512 * block.expansion
self.final = FinalBlock(opt=opt, in_channels=512 * block.expansion)
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
# Zero-initialize the last BN in each residual branch,
# so that the residual branch starts with zeros, and each residual block behaves like an identity.
# This improves the model by 0.2~0.3% according to https://arxiv.org/abs/1706.02677
if zero_init_residual:
for m in self.modules():
if isinstance(m, Bottleneck):
nn.init.constant_(m.bn3.weight, 0)
elif isinstance(m, BasicBlock):
nn.init.constant_(m.bn2.weight, 0)
def __init__(self, opt):
super(ResNet, self).__init__()
depth = opt.depth
if depth in [20, 32, 44, 56, 110, 1202]:
blocktype, self.nettype = 'BasicBlock', 'cifar'
elif depth in [164, 1001]:
blocktype, self.nettype = 'BottleneckBlock', 'cifar'
elif depth in [18, 34]:
blocktype, self.nettype = 'BasicBlock', 'imagenet'
elif depth in [50, 101, 152]:
blocktype, self.nettype = 'BottleneckBlock', 'imagenet'
assert depth in [
20, 32, 44, 56, 110, 1202, 164, 1001, 18, 34, 50, 101, 152
]
if blocktype == 'BasicBlock' and self.nettype == 'cifar':
assert (
depth - 2
) % 6 == 0, 'Depth should be 6n+2, and preferably one of 20, 32, 44, 56, 110, 1202'
n = (depth - 2) // 6
block = BasicBlock
in_planes, out_planes = 16, 64
elif blocktype == 'BottleneckBlock' and self.nettype == 'cifar':
assert (
depth - 2
) % 9 == 0, 'Depth should be 9n+2, and preferably one of 164 or 1001'
n = (depth - 2) // 9
block = BottleneckBlock
in_planes, out_planes = 16, 64
elif blocktype == 'BasicBlock' and self.nettype == 'imagenet':
assert (depth in [18, 34])
num_blocks = [2, 2, 2, 2] if depth == 18 else [3, 4, 6, 3]
block = BasicBlock
in_planes, out_planes = 64, 512 #20, 160
elif blocktype == 'BottleneckBlock' and self.nettype == 'imagenet':
assert (depth in [50, 101, 152])
if depth == 50: num_blocks = [3, 4, 6, 3]
elif depth == 101: num_blocks = [3, 4, 23, 3]
elif depth == 152: num_blocks = [3, 8, 36, 3]
block = BottleneckBlock
in_planes, out_planes = 64, 512
else:
assert (1 == 2)
self.num_classes = opt.num_classes
self.initial = InitialBlock(opt=opt,
out_channels=in_planes,
kernel_size=3,
stride=1,
padding=1)
if self.nettype == 'cifar':
self.group1 = ResidualBlock(opt, block, 16, 16, n, stride=1)
self.group2 = ResidualBlock(opt,
block,
16 * block.expansion,
32,
n,
stride=2)
self.group3 = ResidualBlock(opt,
block,
32 * block.expansion,
64,
n,
stride=2)
elif self.nettype == 'imagenet':
self.group1 = ResidualBlock(
opt, block, 64, 64, num_blocks[0],
stride=1) #For ResNet-S, convert this to 20,20
self.group2 = ResidualBlock(
opt, block, 64 * block.expansion, 128, num_blocks[1],
stride=2) #For ResNet-S, convert this to 20,40
self.group3 = ResidualBlock(
opt,
block,
128 * block.expansion,
256,
num_blocks[2],
stride=2) #For ResNet-S, convert this to 40,80
self.group4 = ResidualBlock(
opt,
block,
256 * block.expansion,
512,
num_blocks[3],
stride=2) #For ResNet-S, convert this to 80,160
else:
assert (1 == 2)
self.pool = nn.AdaptiveAvgPool2d(1)
self.dim_out = out_planes * block.expansion
self.final = FinalBlock(opt=opt,
in_channels=out_planes * block.expansion)
for m in self.modules():
if isinstance(m, nn.Conv2d):
nn.init.kaiming_normal_(m.weight,
mode='fan_out',
nonlinearity='relu')
elif isinstance(m, (nn.BatchNorm2d, nn.GroupNorm)):
nn.init.constant_(m.weight, 1)
nn.init.constant_(m.bias, 0)
