def __init__(self, fc_params, output_params, flatten=False): super(FcBlockWOutput, self).__init__() input_size = fc_params[0] output_size = fc_params[1] add_output = output_params[0] num_classes = output_params[1] self.output_id = output_params[2] self.depth = 1 fc_layers = [] if flatten: fc_layers.append(af.Flatten()) fc_layers.append(nn.Linear(input_size, output_size)) fc_layers.append(nn.ReLU()) fc_layers.append(nn.Dropout(0.5)) self.layers = nn.Sequential(*fc_layers) if add_output: self.output = nn.Linear(output_size, num_classes) self.no_output = False else: self.output = nn.Sequential() self.forward = self.only_forward self.no_output = True
def __init__(self, total_size, ics, init_weights=True): super(DenseNet, self).__init__() self.total_size = total_size self.init_weights = init_weights self.ics = ics self.num_ics = sum(self.ics) self.num_class = 10 self.num_output = 0 self.train_func = mf.iter_training_0 self.test_func = mf.sdn_test self.input_size = 32 self.in_channels = 16 self.cum_in_channels = self.in_channels self.init_conv = nn.Sequential(*[ nn.Conv2d(3, self.in_channels, kernel_size=3, stride=1, padding=1, bias=False), nn.BatchNorm2d(self.in_channels), nn.ReLu() ]) self.end_layers = nn.Sequential(*[ nn.AvgPool2d(kernel_size=8), af.Flatten(), nn.linear(2560, self.num_class) ]) self.grow() if self.init_weights: self._init_weights(self.modules())
def __init__(self, params): super(ResNet_SDN, self).__init__() self.ic_only = False self.num_blocks = params['num_blocks'] self.num_classes = int(params['num_classes']) self.augment_training = params['augment_training'] self.input_size = int(params['input_size']) self.block_type = params['block_type'] self.add_out_nonflat = params['add_ic'] self.add_output = [item for sublist in self.add_out_nonflat for item in sublist] self.init_weights = params['init_weights'] self.train_func = mf.sdn_train self.in_channels = 16 self.num_output = sum(self.add_output) + 1 self.test_func = mf.sdn_test self.init_depth = 1 self.end_depth = 1 self.cur_output_id = 0 if self.block_type == 'basic': self.block = BasicBlockWOutput init_conv = [] if self.input_size == 32: # cifar10 self.cur_input_size = self.input_size init_conv.append(nn.Conv2d(3, self.in_channels, kernel_size=3, stride=1, padding=1, bias=False)) else: # tiny imagenet self.cur_input_size = int(self.input_size / 2) init_conv.append(nn.Conv2d(3, self.in_channels, kernel_size=3, stride=2, padding=1, bias=False)) init_conv.append(nn.BatchNorm2d(self.in_channels)) init_conv.append(nn.ReLU()) self.init_conv = nn.Sequential(*init_conv) self.layers = nn.ModuleList() self.layers.extend(self._make_layer(self.in_channels, block_id=0, stride=1)) self.cur_input_size = int(self.cur_input_size/2) self.layers.extend(self._make_layer(32, block_id=1, stride=2)) self.cur_input_size = int(self.cur_input_size/2) self.layers.extend(self._make_layer(64, block_id=2, stride=2)) end_layers = [] end_layers.append(nn.AvgPool2d(kernel_size=8)) end_layers.append(af.Flatten()) end_layers.append(nn.Linear(16 * self.block.expansion, self.num_classes)) # end_layers.append(nn.Linear(64*self.block.expansion, self.num_classes)) self.end_layers = nn.Sequential(*end_layers) if self.init_weights: self.initialize_weights()
def __init__(self, params): super(WideResNet, self).__init__() self.num_blocks = params['num_blocks'] self.widen_factor = params['widen_factor'] self.num_classes = int(params['num_classes']) self.dropout_rate = params['dropout_rate'] self.augment_training = params['augment_training'] self.input_size = int(params['input_size']) self.train_func = mf.cnn_train self.test_func = mf.cnn_test self.in_channels = 16 self.num_output = 1 if self.input_size == 32: # cifar10 and cifar100 self.init_conv = nn.Conv2d(3, self.in_channels, kernel_size=3, stride=1, padding=1, bias=True) elif self.input_size == 64: # tiny imagenet self.init_conv = nn.Conv2d(3, self.in_channels, kernel_size=3, stride=2, padding=1, bias=True) self.layers = nn.ModuleList() self.layers.extend( self._wide_layer(wide_basic, self.in_channels * self.widen_factor, block_id=0, stride=1)) self.layers.extend( self._wide_layer(wide_basic, 32 * self.widen_factor, block_id=1, stride=2)) self.layers.extend( self._wide_layer(wide_basic, 64 * self.widen_factor, block_id=2, stride=2)) end_layers = [] end_layers.append(nn.BatchNorm2d(64 * self.widen_factor, momentum=0.9)) end_layers.append(nn.ReLU(inplace=True)) end_layers.append(nn.AvgPool2d(kernel_size=8)) end_layers.append(af.Flatten()) end_layers.append(nn.Linear(64 * self.widen_factor, self.num_classes)) self.end_layers = nn.Sequential(*end_layers) self.initialize_weights()
def __init__(self, params): super(ResNet, self).__init__() self.num_blocks = params['num_blocks'] self.num_classes = int(params['num_classes']) self.augment_training = params['augment_training'] self.input_size = int(params['input_size']) self.block_type = params['block_type'] self.train_func = mf.cnn_train self.test_func = mf.cnn_test self.in_channels = 16 self.num_output = 1 if self.block_type == 'basic': self.block = BasicBlock init_conv = [] if self.input_size == 32: # cifar10 and cifar100 init_conv.append( nn.Conv2d(3, self.in_channels, kernel_size=3, stride=1, padding=1, bias=False)) elif self.input_size == 64: # tiny imagenet init_conv.append( nn.Conv2d(3, self.in_channels, kernel_size=3, stride=2, padding=1, bias=False)) init_conv.append(nn.BatchNorm2d(self.in_channels)) init_conv.append(nn.ReLU(inplace=True)) self.init_conv = nn.Sequential(*init_conv) self.layers = nn.ModuleList() self.layers.extend( self._make_layer(self.in_channels, block_id=0, stride=1)) self.layers.extend(self._make_layer(32, block_id=1, stride=2)) self.layers.extend(self._make_layer(64, block_id=2, stride=2)) end_layers = [] end_layers.append(nn.AvgPool2d(kernel_size=8)) end_layers.append(af.Flatten()) end_layers.append( nn.Linear(64 * self.block.expansion, self.num_classes)) self.end_layers = nn.Sequential(*end_layers) self.initialize_weights()
def __init__(self, fc_params, flatten): super(FcBlock, self).__init__() input_size = int(fc_params[0]) output_size = int(fc_params[1]) fc_layers = [] if flatten: fc_layers.append(af.Flatten()) fc_layers.append(nn.Linear(input_size, output_size)) fc_layers.append(nn.ReLU()) fc_layers.append(nn.Dropout(0.5)) self.layers = nn.Sequential(*fc_layers)
def __init__(self, args, x, internal_fm, nz_post, net_size=1, category=200, pd=20, device='cuda', target_channel=128, share=False): super(MNIconfidence, self).__init__() self.share = share # print('Share parameter: ', self.share) self.net_size = net_size self.args = args self.internal_fm = internal_fm self.nz_post = nz_post self.num_class = len(nz_post) self.target_channel = target_channel self.device = device self.pe = af.position_encoding(self.num_class, pd).to(self.device) self.input_shape_list = self._get_input_shapes() self.used_shape_list = self._get_used_shapes() self.smallest_fm_shape = self._get_smallest_shape() self.flatten_size = target_channel * self.smallest_fm_shape * self.smallest_fm_shape self.x_module = nn.Sequential( nn.Conv2d(in_channels=3, out_channels=target_channel, kernel_size=3, padding=1), nn.BatchNorm2d(target_channel), nn.ReLU(), nn.AdaptiveAvgPool2d(self.smallest_fm_shape), af.Flatten(), nn.Linear(self.flatten_size, self.flatten_size)) self.unshared_module_list = nn.ModuleList() for i in range(self.num_class): confidence_module = nn.Sequential( nn.Linear(category + pd + 3, 200 * net_size), nn.ReLU(), nn.Dropout(0.7), nn.Linear(200 * net_size, 100 * net_size), nn.ReLU(), nn.Dropout(0.7)) self.unshared_module_list.append(confidence_module) self.shared_module = nn.Sequential( nn.Linear(category + pd + 3, 200 * net_size), nn.ReLU(), nn.Dropout(0.8), nn.Linear(200 * net_size, 100 * net_size), nn.ReLU(), nn.Dropout(0.8)) self.last_layer = nn.Sequential( nn.Linear(100 * self.num_class * net_size, self.num_class))
def __init__(self, params): super(MobileNet_SDN, self).__init__() self.cfg = params['cfg'] self.num_classes = int(params['num_classes']) self.augment_training = params['augment_training'] self.input_size = int(params['input_size']) self.add_output = params['add_ic'] self.train_func = mf.sdn_train self.test_func = mf.sdn_test self.num_output = sum(self.add_output) + 1 self.in_channels = 32 self.cur_input_size = self.input_size self.init_depth = 1 self.end_depth = 1 self.cur_output_id = 0 init_conv = [] init_conv.append( nn.Conv2d(3, self.in_channels, kernel_size=3, stride=1, padding=1, bias=False)) init_conv.append(nn.BatchNorm2d(self.in_channels)) init_conv.append(nn.ReLU(inplace=True)) self.init_conv = nn.Sequential(*init_conv) self.layers = nn.ModuleList() self.layers.extend(self._make_layers(in_channels=self.in_channels)) end_layers = [] if self.input_size == 32: # cifar10 and cifar100 end_layers.append(nn.AvgPool2d(2)) elif self.input_size == 64: # tiny imagenet end_layers.append(nn.AvgPool2d(4)) end_layers.append(af.Flatten()) end_layers.append(nn.Linear(1024, self.num_classes)) self.end_layers = nn.Sequential(*end_layers)
def __init__(self, params): super(ResNet_Baseline, self).__init__() self.ic_only = False self.augment_training = params['augment_training'] self.init_weights = params['init_weights'] self.block_type = params['block_type'] self.init_type = params['init_type'] self.total_size = params['size'] # the size to reach (number of units) self.ics = params['ics'] if 'ics' in params else [] self.num_ics = sum(self.ics) self.prune = params['prune'] if self.prune: self.keep_ratio = params["keep_ratio"] if 'mode' in params: self.mode = params['mode'] else: self.mode = 0 if self.init_type != 'full' and len(self.ics) != self.total_size: raise ValueError( "final size of network does not match the length of ics array: {}; {}".format(self.total_size, self.ics)) self.num_class = 10 self.num_output = 0 if self.block_type == 'basic': self.block = resNet.BasicBlockWOutput self.input_size = 32 # cifar10 self.in_channels = 16 init_conv = [] init_conv.append(nn.Conv2d(3, self.in_channels, kernel_size=3, stride=1, padding=1, bias=False)) init_conv.append(nn.BatchNorm2d(self.in_channels)) init_conv.append(nn.ReLU()) end_layers = [] end_layers.append(nn.AvgPool2d(kernel_size=8)) end_layers.append(af.Flatten()) if self.init_type == 'dense': end_layers.append(nn.Linear(256 * (self.total_size + 1) * self.block.expansion, self.num_class)) else: end_layers.append(nn.Linear(256 * self.block.expansion, self.num_class)) self.init_conv = nn.Sequential(*init_conv) self.layers = nn.ModuleList() self.end_layers = nn.Sequential(*end_layers) train_funcs = { '0': mf.iter_training_0, '1': mf.iter_training_4 } if self.init_type == "full": self.train_func = mf.cnn_train self.test_func = mf.cnn_test layers = [self.block(self.in_channels, 16, (False, self.num_class, 32, 1)) for _ in range(self.total_size)] self.layers.extend(layers) self.num_output = 1 elif self.init_type == "full_ic": self.train_func = mf.sdn_train self.test_func = mf.sdn_test layers = [self.block(self.in_channels, 16, (self.ics[i], self.num_class, 32, 1)) for i in range(self.total_size)] self.layers.extend(layers) self.num_output = sum(self.ics) + 1 elif self.init_type == "iterative": self.train_func = train_funcs[self.mode] print("mode function: {}".format(self.train_func)) self.test_func = mf.sdn_test self.grow() elif self.init_type == "dense": self.train_func = train_funcs[self.mode] print("mode function: {}".format(self.train_func)) self.test_func = mf.sdn_test self.grow() else: raise KeyError( "the init_type should be either 'full', 'full_ic' or 'iterative' and it is: {}".format(self.init_type)) self.to_eval() if self.init_weights: self._init_weights(self.modules())
def __init__(self, args, x, internal_fm, nz_post, category=200, pd=20, net_size=1, device='cuda', target_channel=128, share=False): super(MulticlassNetImage, self).__init__() self.args = args self.internal_fm = internal_fm self.nz_post = nz_post self.num_class = len(nz_post) self.target_channel = target_channel self.device = device self.pe = af.position_encoding(self.num_class, pd).to(self.device) self.input_shape_list = self._get_input_shapes() self.used_shape_list = self._get_used_shapes() self.smallest_fm_shape = self._get_smallest_shape() self.fm_module_list = nn.ModuleList() self.flatten_size = target_channel * self.smallest_fm_shape * self.smallest_fm_shape self.x_module = nn.Sequential( nn.Conv2d(in_channels=3, out_channels=target_channel, kernel_size=3, padding=1), nn.BatchNorm2d(target_channel), nn.ReLU(), nn.AdaptiveAvgPool2d(self.smallest_fm_shape), af.Flatten(), nn.Linear(self.flatten_size, self.flatten_size)) for shape in self.used_shape_list: if len(shape) == 4: fm_module = nn.Sequential( nn.Conv2d(in_channels=shape[1], out_channels=target_channel, kernel_size=3, padding=1), nn.BatchNorm2d(target_channel), nn.ReLU(), nn.AdaptiveAvgPool2d(self.smallest_fm_shape), af.Flatten(), nn.Linear(self.flatten_size, self.flatten_size)) if len(shape) == 2: fm_module = nn.Sequential( nn.Linear(shape[1], self.flatten_size), nn.ReLU()) self.fm_module_list.append(fm_module) self.total_channels = target_channel * (len(self.used_shape_list) + 1) self.final_module = nn.Sequential( nn.Conv2d(in_channels=self.total_channels, out_channels=target_channel, kernel_size=3, padding=1), nn.BatchNorm2d(target_channel), nn.ReLU(), nn.Dropout(0.8), af.Flatten(), nn.Linear(self.flatten_size, self.num_class)) self.xhs_module_fc = nn.Sequential( nn.Linear(category * self.num_class, category), nn.ReLU(), nn.Dropout(0.7), nn.Linear(category, self.num_class)) self.encoder_layer = nn.TransformerEncoderLayer(category, 4) self.transformer_encoder = nn.TransformerEncoder(self.encoder_layer, 2) self.xhs_module_conv = nn.Sequential( nn.Conv1d(in_channels=category, out_channels=50, kernel_size=1), nn.ReLU(), nn.Dropout(0.7), af.Flatten(), nn.Linear(self.num_class * 50, self.num_class)) self.shared_module = nn.Sequential( nn.Linear(category + pd + 3 + self.flatten_size, 400), nn.ReLU(), nn.Dropout(0.8), nn.Linear(400, 200), nn.ReLU(), nn.Dropout(0.8)) self.shared_module_fm = nn.Sequential( nn.Linear(self.flatten_size * 2 + 3, 400), nn.ReLU(), nn.Dropout(0.8), nn.Linear(400, 200), nn.ReLU(), nn.Dropout(0.8)) self.last_layer = nn.Sequential( nn.Linear(200 * self.num_class, self.num_class))