def __init__(self, args, conv3x3=common.default_conv): super(CFQKBN, self).__init__() n_classes = 10 if args.data_train == 'CIFAR10' else 100 self.conv1 = common.BasicBlock(3, 32, 5, stride=1, bias=False, conv3x3=conv3x3, args=args) self.conv2 = common.BasicBlock(32, 32, 5, stride=1, bias=False, conv3x3=conv3x3, args=args) self.conv3 = common.BasicBlock(32, 64, 5, stride=1, bias=False, conv3x3=conv3x3, args=args) self.fc1 = nn.Linear(in_features=3 * 3 * 64, out_features=64) self.relu = nn.ReLU(inplace=True) self.fc2 = nn.Linear(in_features=64, out_features=n_classes)
def __init__(self, channel=128, reduction=2, ksize=3, scale=3, stride=1, softmax_scale=10, average=True, conv=common.default_conv): super(NonLocalAttention, self).__init__() self.conv_match1 = common.BasicBlock(conv, channel, channel // reduction, 1, bn=False, act=nn.PReLU()) self.conv_match2 = common.BasicBlock(conv, channel, channel // reduction, 1, bn=False, act=nn.PReLU()) self.conv_assembly = common.BasicBlock(conv, channel, channel, 1, bn=False, act=nn.PReLU())
def __init__(self, args, gan_type='GAN'): super(Discriminator, self).__init__() in_channels = 3 out_channels = 64 depth = 3 #bn = not gan_type == 'WGAN_GP' bn = True act = nn.LeakyReLU(negative_slope=0.2, inplace=True) m_features = [ common.BasicBlock(args.n_colors, out_channels, 3, bn=bn, act=act) ] for i in range(depth): in_channels = out_channels if i % 2 == 1: stride = 1 out_channels *= 2 else: stride = 2 m_features.append(common.BasicBlock( in_channels, out_channels, 3, stride=stride, bn=bn, act=act )) self.features = nn.Sequential(*m_features) patch_size = args.patch_size // (2**((depth + 1) // 2)) m_classifier = [ nn.Linear(out_channels * patch_size**2, 1024), act, nn.Linear(1024, 1) ] self.classifier = nn.Sequential(*m_classifier)
def __init__(self, conv=common.default_conv, **kwargs,): super(MSMNetModel, self).__init__() # -------------- Define multi-scale model architecture here ------------ # scale = 3 input_channles = kwargs['input_channels'] num_resblocks = kwargs['num_ms_resblocks'] intermediate_channels = kwargs['intermediate_channels'] kernel_size = kwargs['default_kernel_size'] activation = nn.ReLU(True) rgb_range = kwargs['rgb_range'] self.sub_mean = common.MeanShift(rgb_range) self.add_mean = common.MeanShift(rgb_range, sign=1) # head to read scaled image _head = common.BasicBlock(conv, input_channles, intermediate_channels, kernel_size) # pre-process 2*Resblock each self.pre_process = nn.ModuleList([ nn.Sequential( common.PreResBlock(conv, 2*intermediate_channels, intermediate_channels, 5, bn=True, act=activation), common.ResBlock(conv, intermediate_channels, 5, bn=True, act=activation) ), nn.Sequential( common.PreResBlock(conv, 2*intermediate_channels, intermediate_channels, 5, bn=True, act=activation), common.ResBlock(conv, intermediate_channels, 5, bn=True, act=activation) ), nn.Sequential( common.ResBlock(conv, intermediate_channels, 5, bn=True, act=activation), common.ResBlock(conv, intermediate_channels, 5, bn=True, act=activation) ) ]) # body 16*Resblocks each _body = [ common.ResBlock( conv, intermediate_channels, kernel_size, bn=True, act=activation ) for _ in range(num_resblocks) ] _body.append(conv(intermediate_channels, intermediate_channels, kernel_size)) # upsample to enlarge the scale self.upsample = common.Upsampler(conv, intermediate_channels, bn=False, act=False) _output = nn.ModuleList([ # conv(3, 1, 3), common.BasicBlock(conv, intermediate_channels, intermediate_channels, kernel_size, act=nn.Sigmoid()) ]) self.head = nn.Sequential(*_head) self.body = nn.Sequential(*_body) self.output = nn.Sequential(*_output)
def __init__(self, conv=common.default_conv, **kwargs): super(FusionModel, self).__init__() num_resblocks = kwargs['num_fusion_resblocks'] intermediate_channels = kwargs['intermediate_channels'] res_block = [ common.ResBlock(conv, intermediate_channels, 3, bn=True) for _ in range(num_resblocks) ] fusion_model = nn.ModuleList([ common.BasicBlock(conv, 2*intermediate_channels, intermediate_channels, 3), nn.Sequential(*res_block), common.BasicBlock(conv, intermediate_channels, 1, 3, act=nn.Sigmoid()) ]) self.fusion_model = nn.Sequential(*fusion_model)
def __init__(self, args, conv=common.default_conv): super(MSSR, self).__init__() #n_convblock = args.n_convblocks n_feats = args.n_feats self.depth = args.depth kernel_size = 3 scale = args.scale[0] rgb_mean = (0.4488, 0.4371, 0.4040) rgb_std = (1.0, 1.0, 1.0) self.sub_mean = common.MeanShift(args.rgb_range, rgb_mean, rgb_std) # define head module m_head = [ common.BasicBlock(conv, args.n_colors, n_feats, kernel_size, stride=1, bias=True, bn=False, act=nn.PReLU()), common.BasicBlock(conv, n_feats, n_feats, kernel_size, stride=1, bias=True, bn=False, act=nn.PReLU()) ] # define multiple reconstruction module self.body = RecurrentProjection(n_feats) # define tail module m_tail = [ nn.Conv2d(n_feats * self.depth, args.n_colors, kernel_size, padding=(kernel_size // 2)) ] self.add_mean = common.MeanShift(args.rgb_range, rgb_mean, rgb_std, 1) self.head = nn.Sequential(*m_head) self.tail = nn.Sequential(*m_tail)
def __init__(self, in_channel, kernel_size=3, conv=common.default_conv): super(RecurrentProjection, self).__init__() self.multi_source_projection_1 = MultisourceProjection( in_channel, kernel_size=kernel_size, conv=conv) self.multi_source_projection_2 = MultisourceProjection( in_channel, kernel_size=kernel_size, conv=conv) self.down_sample_1 = nn.Sequential(*[ nn.Conv2d(in_channel, in_channel, 6, stride=2, padding=2), nn.PReLU() ]) #self.down_sample_2 = nn.Sequential(*[nn.Conv2d(in_channel,in_channel,6,stride=2,padding=2),nn.PReLU()]) self.down_sample_3 = nn.Sequential(*[ nn.Conv2d(in_channel, in_channel, 8, stride=4, padding=2), nn.PReLU() ]) self.down_sample_4 = nn.Sequential(*[ nn.Conv2d(in_channel, in_channel, 8, stride=4, padding=2), nn.PReLU() ]) self.error_encode_1 = nn.Sequential(*[ nn.ConvTranspose2d(in_channel, in_channel, 6, stride=2, padding=2), nn.PReLU() ]) self.error_encode_2 = nn.Sequential(*[ nn.ConvTranspose2d(in_channel, in_channel, 8, stride=4, padding=2), nn.PReLU() ]) self.post_conv = common.BasicBlock(conv, in_channel, in_channel, kernel_size, stride=1, bias=True, act=nn.PReLU())
def __init__(self, args, conv3x3=conv_factor, conv1x1=None): super(VGG, self).__init__() args = args[0] # we use batch noramlization for VGG norm = common.default_norm bias = not args.no_bias configs = { 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 'B': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M' ], '16': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M' ], '19': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M' ], 'ef': [ 32, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M' ] } body_list = [] in_channels = args.n_colors for i, v in enumerate(configs[args.vgg_type]): if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: t = 3 if args.vgg_decom_type == 'all' else 8 if i <= t: body_list.append( common.BasicBlock(in_channels, v, args.kernel_size, bias=bias, conv3x3=common.default_conv, norm=norm)) else: body_list.append( BasicBlock(in_channels, v, args.kernel_size, bias=bias, conv=conv3x3, norm=norm)) in_channels = v # for CIFAR10 and CIFAR100 only assert (args.data_train.find('CIFAR') >= 0) n_classes = int(args.data_train[5:]) self.features = nn.Sequential(*body_list) self.classifier = nn.Linear(in_channels, n_classes)
def __init__(self, conv=common.default_conv, **kwargs): super(AttentionModel, self).__init__() kernel_size = kwargs['default_kernel_size'] intermediate_channels = kwargs['intermediate_channels'] attention_input_channels = kwargs['attention_input_channels'] dense_growth_rate = kwargs['dense_growth_rate'] # -------------- Define attention model here ---------------------- self.attention_conv_head = nn.Sequential( common.BasicBlock(conv, attention_input_channels, intermediate_channels, kernel_size), common.BasicBlock(conv, intermediate_channels, intermediate_channels, kernel_size), ) self.attention_maxpool = nn.MaxPool2d(kernel_size=2, return_indices=True) self.down_dense_1 = common.DenseBlock(3, intermediate_channels, bn_size=4, growth_rate=dense_growth_rate, drop_rate=0) down_dense_1_output_channels = intermediate_channels + 3*dense_growth_rate self.down_dense_1_trans = common.Transition(down_dense_1_output_channels, down_dense_1_output_channels // 2) self.down_dense_2 = common.DenseBlock(3, down_dense_1_output_channels // 2, bn_size=4, growth_rate=dense_growth_rate, drop_rate=0) down_dense_2_output_channels = down_dense_1_output_channels // 2 + 3*dense_growth_rate self.down_dense_2_trans = common.Transition(down_dense_2_output_channels, down_dense_2_output_channels // 2) self.bottom_dense = common.DenseBlock(3, down_dense_2_output_channels // 2, bn_size=4, growth_rate=dense_growth_rate, drop_rate=0) bottom_dense_output_channels = down_dense_2_output_channels // 2 + 3*dense_growth_rate self.bottom_dense_upsample = common.Upsampler(conv, bottom_dense_output_channels) self.up_dense_2 = common.DenseBlock(3, bottom_dense_output_channels+down_dense_2_output_channels, bn_size=4, growth_rate=dense_growth_rate, drop_rate=0) up_dense_2_output_channels = bottom_dense_output_channels \ + down_dense_2_output_channels \ + 3*dense_growth_rate self.up_dense_2_upsample = common.Upsampler(conv, up_dense_2_output_channels) self.up_dense_1 = common.DenseBlock(3, up_dense_2_output_channels+down_dense_1_output_channels, bn_size=4, growth_rate=dense_growth_rate, drop_rate=0) up_dense_1_output_channels = up_dense_2_output_channels \ + down_dense_1_output_channels \ + 3*dense_growth_rate self.up_dense_1_upsample = common.Upsampler(conv, up_dense_1_output_channels) self.attention_conv_tail = nn.Sequential( common.BasicBlock(conv, up_dense_1_output_channels+intermediate_channels, intermediate_channels, kernel_size), common.BasicBlock(conv, intermediate_channels, intermediate_channels, kernel_size, act=nn.Sigmoid()) )
def basic_block(in_channels, out_channels, act): return common.BasicBlock(conv, in_channels, out_channels, kernel_size, bias=True, bn=False, act=act)
def __init__(self, level=5, res_scale=1, channel=64, reduction=2, ksize=3, stride=1, softmax_scale=10, average=True, conv=common.default_conv): super(PyramidAttention, self).__init__() self.ksize = ksize self.stride = stride self.res_scale = res_scale self.softmax_scale = softmax_scale self.scale = [1 - i / 10 for i in range(level)] self.average = average escape_NaN = torch.FloatTensor([1e-4]) self.register_buffer('escape_NaN', escape_NaN) self.conv_match_L_base = common.BasicBlock(conv, channel, channel // reduction, 1, bn=False, act=nn.PReLU()) self.conv_match = common.BasicBlock(conv, channel, channel // reduction, 1, bn=False, act=nn.PReLU()) self.conv_assembly = common.BasicBlock(conv, channel, channel, 1, bn=False, act=nn.PReLU())
def __init__(self, channel=128, reduction=2, ksize=3, scale=3, stride=1, softmax_scale=10, average=True, conv=common.default_conv): super(CrossScaleAttention, self).__init__() self.ksize = ksize self.stride = stride self.softmax_scale = softmax_scale self.scale = scale self.average = average escape_NaN = torch.FloatTensor([1e-4]) self.register_buffer('escape_NaN', escape_NaN) self.conv_match_1 = common.BasicBlock(conv, channel, channel // reduction, 1, bn=False, act=nn.PReLU()) self.conv_match_2 = common.BasicBlock(conv, channel, channel // reduction, 1, bn=False, act=nn.PReLU()) self.conv_assembly = common.BasicBlock(conv, channel, channel, 1, bn=False, act=nn.PReLU())
def __init__(self, args, conv3x3=common.default_conv, conv1x1=common.default_conv): super(ResNet, self).__init__() n_classes = int( args.data_train[5:]) if args.data_train.find('CIFAR') >= 0 else 200 kernel_size = args.kernel_size if args.depth == 50: self.expansion = 4 self.block = BottleNeck self.n_blocks = (args.depth - 2) // 9 # elif args.depth <= 56: else: self.expansion = 1 self.block = ResBlock self.n_blocks = (args.depth - 2) // 6 # else: # self.expansion = 4 # self.block = BottleNeck # self.n_blocks = (args.depth - 2) // 9 self.in_channels = 16 self.downsample_type = args.downsample_type bias = not args.no_bias kwargs = {'conv3x3': conv3x3, 'conv1x1': conv1x1, 'args': args} stride = 1 if args.data_train.find('CIFAR') >= 0 else 2 m = [ common.BasicBlock(args.n_colors, 16, kernel_size=kernel_size, stride=stride, bias=bias, conv3x3=conv3x3, args=args), self.make_layer(self.n_blocks, 16, kernel_size, **kwargs), self.make_layer(self.n_blocks, 32, kernel_size, stride=2, **kwargs), self.make_layer(self.n_blocks, 64, kernel_size, stride=2, **kwargs), nn.AvgPool2d(8) ] fc = nn.Linear(64 * self.expansion, n_classes) self.features = nn.Sequential(*m) self.classifier = fc
def __init__(self, args, conv3x3=common.default_conv, conv1x1=None): super(VGG, self).__init__() # we use batch noramlization for VGG norm = None self.norm = norm bias = not args.no_bias configs = { 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], '16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], '19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 'ef': [32, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M'] } body_list = [] in_channels = args.n_colors if args.data_train.find('CIFAR') >= 0 or args.data_train.find('Tiny') >= 0: for i, v in enumerate(configs[args.vgg_type]): if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: stride = 2 if i == 0 and args.data_train.find('Tiny') >= 0 else 1 body_list.append(common.BasicBlock(in_channels, v, args.kernel_size, stride=stride, bias=bias, conv3x3=conv3x3)) in_channels = v else: for i, v in enumerate(configs[args.vgg_type]): if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: conv2d = conv3x3(in_channels, v, kernel_size=3) if norm is not None: body_list += [conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True)] else: body_list += [conv2d, nn.ReLU(inplace=True)] in_channels = v # assert(args.data_train.find('CIFAR') >= 0) self.body_list = body_list self.features = nn.Sequential(*body_list) if args.data_train.find('CIFAR') >= 0: n_classes = int(args.data_train[5:]) if args.template.find('linear3') >= 0: self.classifier = nn.Sequential(nn.Linear(in_channels, in_channels), nn.Linear(in_channels, in_channels), nn.Linear(in_channels, n_classes)) else: self.classifier = nn.Linear(in_channels, n_classes) if args.data_train.find('Tiny') >= 0: n_classes = 200 self.classifier = nn.Sequential(nn.Linear(in_channels, in_channels), nn.Linear(in_channels, in_channels), nn.Linear(in_channels, n_classes)) elif args.data_train == 'ImageNet': n_classes = 1000 self.classifier = nn.Sequential( nn.Linear(512 * 7 * 7, 4096), nn.ReLU(True), nn.Dropout(), nn.Linear(4096, 4096), nn.ReLU(True), nn.Dropout(), nn.Linear(4096, n_classes), ) # self.classifier = nn.Sequential( # nn.Linear(512 * 7 * 7, n_classes) # ) # print(conv3x3, conv3x3 == common.default_conv or conv3x3 == nn.Conv2d) # if conv3x3 == common.default_conv or conv3x3 == nn.Conv2d: # self.load(args, strict=True) self.total_time = [0] * len(body_list) self.top1_err_list = [] self.sum_list = [] self.layer_num = -1 # Cluster Prunning实验层编号 self.spec_list = [] # 第layer_num层推理时间 self.current_ratio_list = [] # flops率 self.parameter_ratio_list = [] # parameter率 self.timer_test_list = [] # 整个网络推理时间 self.block_dict = {0: 0, 1: 1, 3: 2, 4: 3, 6: 4, 7: 5, 8: 6, 10: 7, 11: 8, 12: 9, 14: 10, 15: 11, 16: 12} for i in range(len(body_list)): body_list[i] = nn.Sequential(body_list[i])
def __init__(self, in_channel, kernel_size=3, scale=2, conv=common.default_conv): super(RecurrentProjection, self).__init__() self.scale = scale stride_conv_ksize, stride, padding = { 2: (6, 2, 2), 3: (9, 3, 3), 4: (6, 2, 2) }[scale] self.multi_source_projection = MultisourceProjection( in_channel, kernel_size=kernel_size, scale=scale, conv=conv) self.down_sample_1 = nn.Sequential(*[ nn.Conv2d(in_channel, in_channel, stride_conv_ksize, stride=stride, padding=padding), nn.PReLU() ]) if scale != 4: self.down_sample_2 = nn.Sequential(*[ nn.Conv2d(in_channel, in_channel, stride_conv_ksize, stride=stride, padding=padding), nn.PReLU() ]) self.error_encode = nn.Sequential(*[ nn.ConvTranspose2d(in_channel, in_channel, stride_conv_ksize, stride=stride, padding=padding), nn.PReLU() ]) self.post_conv = common.BasicBlock(conv, in_channel, in_channel, kernel_size, stride=1, bias=True, act=nn.PReLU()) if scale == 4: self.multi_source_projection_2 = MultisourceProjection( in_channel, kernel_size=kernel_size, scale=scale, conv=conv) self.down_sample_3 = nn.Sequential(*[ nn.Conv2d(in_channel, in_channel, 8, stride=4, padding=2), nn.PReLU() ]) self.down_sample_4 = nn.Sequential(*[ nn.Conv2d(in_channel, in_channel, 8, stride=4, padding=2), nn.PReLU() ]) self.error_encode_2 = nn.Sequential(*[ nn.ConvTranspose2d( in_channel, in_channel, 8, stride=4, padding=2), nn.PReLU() ])
def __init__(self, args, conv3x3=common.default_conv, conv1x1=None): super(VGG_Basis, self).__init__() # we use batch noramlization for VGG args = args[0] norm = common.default_norm bias = not args.no_bias n_basis = args.n_basis basis_size = args.basis_size configs = { 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 'B': [64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], '16': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M'], '19': [64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M'], 'ef': [32, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M'] } body_list = [] in_channels = args.n_colors for i, v in enumerate(configs[args.vgg_type]): if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: t = 3 if args.vgg_decom_type == 'all' else 8 if i <= t: body_list.append(common.BasicBlock(in_channels, v, args.kernel_size, bias=bias, conv3x3=conv3x3, norm=norm)) else: body_list.append(BasicBlock(in_channels, v, n_basis, basis_size, args.kernel_size, bias=bias, conv=conv3x3, norm=norm)) in_channels = v # assert(args.data_train.find('CIFAR') >= 0) self.features = nn.Sequential(*body_list) if args.data_train.find('CIFAR') >= 0: n_classes = int(args.data_train[5:]) self.classifier = nn.Linear(in_channels, n_classes) elif args.data_train == 'ImageNet': n_classes = 1000 self.classifier = nn.Sequential( nn.Linear(512 * 7 * 7, 4096), nn.ReLU(True), nn.Dropout(), nn.Linear(4096, 4096), nn.ReLU(True), nn.Dropout(), nn.Linear(4096, n_classes), ) if conv3x3 == common.default_conv: model_dir = os.path.join('..', 'models') os.makedirs(model_dir, exist_ok=True) if args.data_train.find('CIFAR') >= 0: if args.pretrained == 'download' or args.extend == 'download': url = ( 'https://cv.snu.ac.kr/' 'research/clustering_kernels/models/vgg16-89711a85.pt' ) state = model_zoo.load_url(url, model_dir=model_dir) elif args.extend: state = torch.load(args.extend) else: common.init_vgg(self) return elif args.data_train == 'ImageNet': if args.pretrained == 'download': url = 'https://download.pytorch.org/models/vgg16_bn-6c64b313.pth' state = model_zoo.load_url(url, model_dir=model_dir) else: common.init_vgg(self) return else: raise NotImplementedError('Unavailable dataset {}'.format(args.data_train)) # from IPython import embed; embed() self.load_state_dict(state, False)
def __init__(self, args, conv3x3=common.default_conv, conv1x1=common.default_conv): super(ResNet_Group, self).__init__() args = args[0] self.args = args m = [] if args.data_train.find('CIFAR') >= 0: self.expansion = 1 self.n_blocks = (args.depth - 2) // 6 self.in_channels = 16 self.downsample_type = 'A' n_classes = int(args.data_train[5:]) kwargs = { 'kernel_size': args.kernel_size, 'conv3x3': conv3x3, } m.append(common.BasicBlock(args.n_colors, 16, **kwargs)) kwargs['conv3x3'] = BasicBlock m.append(self.make_layer(16, self.n_blocks, **kwargs)) m.append(self.make_layer(32, self.n_blocks, stride=2, **kwargs)) m.append(self.make_layer(64, self.n_blocks, stride=2, **kwargs)) m.append(nn.AvgPool2d(8)) fc = nn.Linear(64 * self.expansion, n_classes) elif args.data_train == 'ImageNet': block_config = { 18: ([2, 2, 2, 2], ResBlock, 1), 34: ([3, 4, 6, 3], ResBlock, 1), 50: ([3, 4, 6, 3], BottleNeck, 4), 101: ([3, 4, 23, 3], BottleNeck, 4), 152: ([3, 8, 36, 3], BottleNeck, 4) } n_blocks, self.block, self.expansion = block_config[args.depth] self.in_channels = 64 self.downsample_type = 'C' n_classes = 1000 kwargs = { 'conv3x3': conv3x3, 'conv1x1': conv1x1, } m.append( common.BasicBlock(args.n_colors, 64, 7, stride=2, conv3x3=conv3x3, bias=False)) m.append(nn.MaxPool2d(3, 2, padding=1)) m.append(self.make_layer(64, n_blocks[0], 3, **kwargs)) m.append(self.make_layer(128, n_blocks[1], 3, stride=2, **kwargs)) m.append(self.make_layer(256, n_blocks[2], 3, stride=2, **kwargs)) m.append(self.make_layer(512, n_blocks[3], 3, stride=2, **kwargs)) m.append(nn.AvgPool2d(7, 1)) fc = nn.Linear(512 * self.expansion, n_classes) self.features = nn.Sequential(*m) self.classifier = fc # only if when it is child model if conv3x3 == common.default_conv: if args.pretrained == 'download' or args.extend == 'download': state = getattr(models, 'resnet{}'.format(args.depth))(pretrained=True) elif args.extend: state = torch.load(args.extend) else: common.init_kaiming(self) return source = state.state_dict() target = self.state_dict() for s, t in zip(source.keys(), target.keys()): target[t].copy_(source[s])
def __init__(self, args, conv3x3=common.default_conv): super(ResNet_Basis_Blockwise, self).__init__() args = args[0] self.args = args m = [] if args.data_train.find('CIFAR') >= 0: self.expansion = 1 self.n_basis1 = args.n_basis1 self.n_basis2 = args.n_basis2 self.n_basis3 = args.n_basis3 self.basis_size1 = args.basis_size1 self.basis_size2 = args.basis_size2 self.basis_size3 = args.basis_size3 self.n_blocks = (args.depth - 2) // 6 self.k_size = args.kernel_size self.in_channels = 16 self.downsample_type = 'A' self.basis1 = nn.Parameter( nn.init.kaiming_uniform_( torch.Tensor(self.n_basis1, self.basis_size1, self.k_size, self.k_size))) self.basis2 = nn.Parameter( nn.init.kaiming_uniform_( torch.Tensor(self.n_basis2, self.basis_size2, self.k_size, self.k_size))) self.basis3 = nn.Parameter( nn.init.kaiming_uniform_( torch.Tensor(self.n_basis3, self.basis_size3, self.k_size, self.k_size))) self.block = ResBlockDecom n_classes = int(args.data_train[5:]) kwargs = {'kernel_size': args.kernel_size, 'conv3x3': conv3x3} m.append(common.BasicBlock(args.n_colors, 16, **kwargs)) kwargs['conv3x3'] = BasicBlock kwargs['kernel_size2'] = args.k_size2 kwargs['n_basis'] = self.n_basis1 kwargs['basis_size'] = self.basis_size1 kwargs['basis_l1'] = self.basis1 kwargs['basis_l2'] = self.basis1 m.append(self.make_layer(16, self.n_blocks, **kwargs)) kwargs['n_basis'] = self.n_basis2 kwargs['basis_size'] = self.basis_size2 kwargs['basis_l1'] = self.basis1 kwargs['basis_l2'] = self.basis2 m.append(self.make_layer(32, self.n_blocks, stride=2, **kwargs)) kwargs['n_basis'] = self.n_basis3 kwargs['basis_size'] = self.basis_size3 kwargs['basis_l1'] = self.basis2 kwargs['basis_l2'] = self.basis3 m.append(self.make_layer(64, self.n_blocks, stride=2, **kwargs)) m.append(nn.AvgPool2d(8)) fc = nn.Linear(64 * self.expansion, n_classes) self.features = nn.Sequential(*m) self.classifier = fc # only if when it is child model if conv3x3 == common.default_conv: if args.pretrained == 'download' or args.extend == 'download': state = getattr(models, 'resnet{}'.format(args.depth))(pretrained=True) elif args.extend: state = torch.load(args.extend) else: common.init_kaiming(self) return source = state.state_dict() target = self.state_dict() for s, t in zip(source.keys(), target.keys()): target[t].copy_(source[s])
def __init__(self, args, conv3x3=common.default_conv, conv1x1=None): super(VGG, self).__init__() # args = args[0] # we use batch noramlization for VGG norm = None self.norm = norm bias = not args.no_bias configs = { 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 'B': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M' ], '16': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M' ], '19': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M' ], 'ef': [ 32, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M' ] } body_list = [] in_channels = args.n_colors if args.data_train.find('CIFAR') >= 0: for v in configs[args.vgg_type]: if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: body_list.append( common.BasicBlock(in_channels, v, args.kernel_size, bias=bias, conv3x3=conv3x3, norm=norm)) in_channels = v else: for i, v in enumerate(configs[args.vgg_type]): if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: conv2d = conv3x3(in_channels, v, kernel_size=3) if norm is not None: body_list += [ conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True) ] else: body_list += [conv2d, nn.ReLU(inplace=True)] in_channels = v # assert(args.data_train.find('CIFAR') >= 0) self.features = nn.Sequential(*body_list) if args.data_train.find('CIFAR') >= 0: n_classes = int(args.data_train[5:]) self.classifier = nn.Linear(in_channels, n_classes) elif args.data_train == 'ImageNet': n_classes = 1000 self.classifier = nn.Sequential( nn.Linear(512 * 7 * 7, 4096), nn.ReLU(True), nn.Dropout(), nn.Linear(4096, 4096), nn.ReLU(True), nn.Dropout(), nn.Linear(4096, n_classes), ) # self.classifier = nn.Sequential( # nn.Linear(512 * 7 * 7, n_classes) # ) print(conv3x3, conv3x3 == common.default_conv or conv3x3 == nn.Conv2d) if conv3x3 == common.default_conv or conv3x3 == nn.Conv2d: self.load(args, strict=True)
def __init__(self, args, conv3x3=common.default_conv, conv1x1=None): super(VGG_GROUP, self).__init__() args = args[0] # we use batch noramlization for VGG norm = common.default_norm bias = not args.no_bias group_size = args.group_size configs = { 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 'B': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M' ], '16': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M' ], '19': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M' ], 'ef': [ 32, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M' ] } body_list = [] in_channels = args.n_colors for i, v in enumerate(configs[args.vgg_type]): if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: t = 3 if args.vgg_decom_type == 'all' else 8 if i <= t: body_list.append( common.BasicBlock(in_channels, v, args.kernel_size, bias=bias, conv3x3=conv3x3, norm=norm)) else: body_list.append( BasicBlock(in_channels, v, group_size, args.kernel_size, bias=bias, conv=conv3x3, norm=norm)) in_channels = v # for CIFAR10 and CIFAR100 only assert (args.data_train.find('CIFAR') >= 0) n_classes = int(args.data_train[5:]) self.features = nn.Sequential(*body_list) self.classifier = nn.Linear(in_channels, n_classes) if conv3x3 == common.default_conv: if args.pretrained == 'download' or args.extend == 'download': url = ('https://cv.snu.ac.kr/' 'research/clustering_kernels/models/vgg16-89711a85.pt') model_dir = os.path.join('..', 'models') os.makedirs(model_dir, exist_ok=True) state = torch.utils.model_zoo.load_url(url, model_dir=model_dir) elif args.extend: state = torch.load(args.extend) else: common.init_vgg(self) return self.load_state_dict(state, strict=False)
def __init__(self, args, conv3x3=common.default_conv, conv1x1=None): super(VGG, self).__init__() # we use batch noramlization for VGG norm = None self.norm = norm bias = not args.no_bias if args.overparam_type == '1': conv = common_tmp.conv3x3_1 elif args.overparam_type == '2': conv = common_tmp.conv3x3_2 elif args.overparam_type == '3': conv = common_tmp.conv3x3_3 elif args.overparam_type == '4': conv = common_tmp.conv3x3_4 elif args.overparam_type == '4tensor1': conv = common_tmp.conv3x3_4tensor1 elif args.overparam_type == '4tensor2': conv = common_tmp.conv3x3_4tensor2 elif args.overparam_type == '4tensor3': conv = common_tmp.conv3x3_4tensor3 elif args.overparam_type == '5': conv = common_tmp.conv3x3_5 elif args.overparam_type == '5relu': conv = common_tmp.conv3x3_5relu elif args.overparam_type == '5bias1': conv = common_tmp.conv3x3_5bias1 elif args.overparam_type == '5bias2': conv = common_tmp.conv3x3_5bias2 elif args.overparam_type == 'R1': conv = common.ROPConv1 elif args.overparam_type == 'R2': conv = common.ROPConv2 else: raise NotImplementedError configs = { 'A': [64, 'M', 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M'], 'B': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 'M', 512, 512, 'M', 512, 512, 'M' ], '16': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 512, 512, 512, 'M', 512, 512, 512, 'M' ], '19': [ 64, 64, 'M', 128, 128, 'M', 256, 256, 256, 256, 'M', 512, 512, 512, 512, 'M', 512, 512, 512, 512, 'M' ], 'ef': [ 32, 64, 'M', 128, 128, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M', 256, 256, 256, 'M' ] } body_list = [] in_channels = args.n_colors if args.data_train.find('CIFAR') >= 0 or args.data_train.find( 'Tiny') >= 0: for i, v in enumerate(configs[args.vgg_type]): if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: stride = 2 if i == 0 and args.data_train.find( 'Tiny') >= 0 else 1 body_list.append( common.BasicBlock(in_channels, v, args.kernel_size, stride=stride, bias=bias, conv3x3=conv)) in_channels = v else: for i, v in enumerate(configs[args.vgg_type]): if v == 'M': body_list.append(nn.MaxPool2d(kernel_size=2, stride=2)) else: conv2d = conv3x3(in_channels, v, kernel_size=3) if norm is not None: body_list += [ conv2d, nn.BatchNorm2d(v), nn.ReLU(inplace=True) ] else: body_list += [conv2d, nn.ReLU(inplace=True)] in_channels = v # assert(args.data_train.find('CIFAR') >= 0) self.features = nn.Sequential(*body_list) if args.data_train.find('CIFAR') >= 0: n_classes = int(args.data_train[5:]) if args.template.find('linear3') >= 0: self.classifier = nn.Sequential( nn.Linear(in_channels, in_channels), nn.Linear(in_channels, in_channels), nn.Linear(in_channels, n_classes)) else: self.classifier = nn.Linear(in_channels, n_classes) if args.data_train.find('Tiny') >= 0: n_classes = 200 self.classifier = nn.Sequential( nn.Linear(in_channels, in_channels), nn.Linear(in_channels, in_channels), nn.Linear(in_channels, n_classes)) elif args.data_train == 'ImageNet': n_classes = 1000 self.classifier = nn.Sequential( nn.Linear(512 * 7 * 7, 4096), nn.ReLU(True), nn.Dropout(), nn.Linear(4096, 4096), nn.ReLU(True), nn.Dropout(), nn.Linear(4096, n_classes), ) # self.classifier = nn.Sequential( # nn.Linear(512 * 7 * 7, n_classes) # ) # print(conv3x3, conv3x3 == common.default_conv or conv3x3 == nn.Conv2d) if conv3x3 == common.default_conv or conv3x3 == nn.Conv2d: self.load(args, strict=True)
def __init__(self, args, conv3x3=common.default_conv, conv1x1=common.default_conv): super(ResNet, self).__init__() m = [] if args.data_train.find('CIFAR') >= 0: self.block = ResBlock self.expansion = 1 self.n_blocks = (args.depth - 2) // 6 self.in_channels = 16 self.downsample_type = 'A' n_classes = int(args.data_train[5:]) kwargs = { 'kernel_size': args.kernel_size, 'conv3x3': conv3x3, } m.append(common.BasicBlock(args.n_colors, 16, **kwargs)) m.append(self.make_layer(16, self.n_blocks, **kwargs)) m.append(self.make_layer(32, self.n_blocks, stride=2, **kwargs)) m.append(self.make_layer(64, self.n_blocks, stride=2, **kwargs)) m.append(nn.AvgPool2d(8)) fc = nn.Linear(64 * self.expansion, n_classes) elif args.data_train == 'ImageNet': block_config = { 18: ([2, 2, 2, 2], ResBlock, 1), 34: ([3, 4, 6, 3], ResBlock, 1), 50: ([3, 4, 6, 3], BottleNeck, 4), 101: ([3, 4, 23, 3], BottleNeck, 4), 152: ([3, 8, 36, 3], BottleNeck, 4) } n_blocks, self.block, self.expansion = block_config[args.depth] self.in_channels = 64 self.downsample_type = 'C' n_classes = 1000 kwargs = { 'conv3x3': conv3x3, 'conv1x1': conv1x1, } m.append( common.BasicBlock(args.n_colors, 64, 7, stride=2, conv3x3=conv3x3, bias=False)) m.append(nn.MaxPool2d(3, 2, padding=1)) m.append(self.make_layer(64, n_blocks[0], 3, **kwargs)) m.append(self.make_layer(128, n_blocks[1], 3, stride=2, **kwargs)) m.append(self.make_layer(256, n_blocks[2], 3, stride=2, **kwargs)) m.append(self.make_layer(512, n_blocks[3], 3, stride=2, **kwargs)) m.append(nn.AvgPool2d(7, 1)) fc = nn.Linear(512 * self.expansion, n_classes) self.features = nn.Sequential(*m) self.classifier = fc # only if when it is child model if conv3x3 == common.default_conv: self.load(args, strict=True)