def layer_test_helper_flatten_1d(layer, channel_index): # This should test that the output is the correct shape so it should pass # into a Dense layer rather than a Conv layer. # The weighted layer is the previous layer, # Create model main_input = Input(shape=list(random.randint(10, 20, size=2))) x = Conv1D(3, 3)(main_input) x = layer(x) x = Flatten()(x) main_output = Dense(5)(x) model = Model(inputs=main_input, outputs=main_output) # Delete channels del_layer_index = 1 next_layer_index = 4 del_layer = model.layers[del_layer_index] surgeon = Surgeon(model) surgeon.add_job("delete_channels", del_layer, channels=channel_index) new_model = surgeon.operate() new_w = new_model.layers[next_layer_index].get_weights() # Calculate next layer's correct weights flat_sz = np.prod(layer.get_output_shape_at(0)[1:]) channel_count = getattr(del_layer, utils.get_channels_attr(del_layer)) channel_index = [i % channel_count for i in channel_index] delete_indices = [ x + i for i in range(0, flat_sz, channel_count) for x in channel_index ] correct_w = model.layers[next_layer_index].get_weights() correct_w[0] = np.delete(correct_w[0], delete_indices, axis=0) assert weights_equal(correct_w, new_w)
def test_delete_channels_downstream_sharing(): # Create all model layers input_1 = Input(shape=(5,)) dense_1 = Dense(4, name='dense_1') dense_2 = Dense(4, name='dense_2') dense_3 = Dense(3, name='dense_3') # Create the base model x = dense_1(input_1) y = dense_2(input_1) output_1 = dense_3(x) output_2 = dense_3(y) model_1 = utils.clean_copy(Model(input_1, [output_1, output_2])) # Delete channels from dense_1 and dense_2 surgeon = Surgeon(model_1) surgeon.add_job('delete_channels', model_1.get_layer(dense_1.name), channels=[0]) surgeon.add_job('delete_channels', model_1.get_layer(dense_2.name), channels=[1]) model_2 = surgeon.operate() # Create the expected model # input_1 = Input(shape=(5,)) dense_1_exp = Dense(3, name='dense_1') dense_2_exp = Dense(3, name='dense_2') dense_3_exp = Dense(3, name='dense_3') # Create the base model x = dense_1_exp(input_1) y = dense_2_exp(input_1) output_1 = dense_3_exp(x) output_2 = dense_3_exp(y) model_2_exp = utils.clean_copy(Model(input_1, [output_1, output_2])) config_1 = model_2.get_config() config_2 = model_2_exp.get_config() config_2['name'] = config_1['name'] # make the config names identical assert json.dumps(config_1) == json.dumps(config_2)
def test_delete_all_channels_in_long_branch(): input_1 = Input(shape=(20, 20, 3)) conv_1 = Conv2D(2, [3, 3], name="conv_1") conv_2 = Conv2D(3, [3, 3], name="conv_2") conv_3 = Conv2D(4, [1, 1], name="conv_3") cat_1 = Concatenate(name="cat_1") x = conv_1(input_1) x = conv_3(x) y = conv_2(input_1) output_1 = cat_1([x, y]) model_1 = utils.clean_copy(Model(input_1, output_1)) surgeon = Surgeon(model_1, copy=True) surgeon.add_job("delete_channels", model_1.get_layer("conv_1"), channels=[0, 1]) model_2 = surgeon.operate() output_1 = conv_2(input_1) model_2_exp = utils.clean_copy(Model(input_1, output_1)) config_1 = model_2.get_config() config_2 = model_2_exp.get_config() config_2["name"] = config_1["name"] # make the config names identical assert config_1 == config_2
def mask_pruning(self, Vec): vec = [] for idx, i in enumerate(Vec): if i == 1: vec.append( idx ) # appendo nel vettore vec gli indici dei filtri da eliminare #this function takes a vector of indexes, if there is a 1 in the j-th position the j-th filter of the network will be pruned. from kerassurgeon import Surgeon # k surgeon = Surgeon(self.model) layer_list = self.model.layers # list of layers in the model e.g [<keras.engine.input_layer.InputLayer at 0x2068f6c2048>,... to_prune_list = [] for i in range(len(layer_list)): if 'conv' in str(layer_list[i]): to_prune_list.append( i ) # check the indexes of the convolutional layers in the model n_filters = [] # print(to_prune_list) for i in to_prune_list: n_filters.append( self.model.get_layer(index=i).get_config()['filters'] ) # gets the number of filter in each convolutional layer # print('num,er of filters=',n_filters) # return n_filters sumlist = [0 for i in n_filters ] # vettore lungo quanto il numero di strati for idx in range(len(n_filters)): sumlist[idx] = sum(n_filters[0:idx + 1]) #[64,64,128]->[64,128,256] # print(sumlist) # print('vec=',vec) mega_list = [[], [], [], [], [], [], [], [], [], [], [], [], []] for num in vec: for idx, i in enumerate(sumlist): if num - i < 0: strato = idx if idx == 0: numero = num else: numero = num - sumlist[idx - 1] # print('strato=',idx,'numero=',numero) mega_list[idx].append(numero) break # print('dimensione di mega list=',len(mega_list)) for kk in range( len(to_prune_list)): # iterate on each convolutional layer # of the convolutional filters layerr = self.model.layers[to_prune_list[kk]] # print('len di mega_list[kk]=',len(mega_list[kk])) surgeon.add_job('delete_channels', layerr, channels=mega_list[kk]) self.model = surgeon.operate() return 0
def semi_random_pruning(self,P,kk): from kerassurgeon import Surgeon layer_list=self.model.layers # list of layers in the model e.g [<keras.engine.input_layer.InputLayer at 0x2068f6c2048>,... to_prune_list=[] for i in range(len(layer_list)): if 'Conv2D' in str(layer_list[i]): to_prune_list.append(i) # check the indexes of the convolutional layers in the model n_filters=[] # print(to_prune_list) for i in to_prune_list: n_filters.append(self.model.get_layer(index=i).get_config()['filters']) # gets the number of filter in each convolutional layer # print('num,er of filters=',n_filters) # return n_filters mega_list=[] # for kk in range(len(to_prune_list)): # iterate on each convolutional layer num_filter_layer=n_filters[kk]# number of filters in the current layer, which is to_prune_list[kk] random_index=[] # index of the filters to be pruned for jj in range(int(np.ceil(P*num_filter_layer))): # P% of the filters tmp=np.random.randint(0,num_filter_layer) while tmp in random_index: tmp=np.random.randint(0,num_filter_layer) # if the filter has alread benn choosen, try again random_index.append(tmp) # mega_list.append(random_index) # contain a list of all the random index of that layer, the length of the mega list is thus the length # of the convolutional filters layerr=self.model.layers[to_prune_list[kk]] print(str(layerr)) surgeon = Surgeon(self.model) surgeon.add_job('delete_channels', layerr, channels=random_index) self.model = surgeon.operate()
def test_delete_channels_merge_concatenate(channel_index, data_format): # This should test that the output is the correct shape so it should pass # into a Dense layer rather than a Conv layer. # The weighted layer is the previous layer, # Create model if data_format == "channels_first": axis = 1 elif data_format == "channels_last": axis = -1 else: raise ValueError input_shape = list(random.randint(10, 20, size=3)) input_1 = Input(shape=input_shape) input_2 = Input(shape=input_shape) x = Conv2D(3, [3, 3], data_format=data_format, name="conv_1")(input_1) y = Conv2D(3, [3, 3], data_format=data_format, name="conv_2")(input_2) x = Concatenate(axis=axis, name="cat_1")([x, y]) x = Flatten()(x) main_output = Dense(5, name="dense_1")(x) model = Model(inputs=[input_1, input_2], outputs=main_output) old_w = model.get_layer("dense_1").get_weights() # Delete channels layer = model.get_layer("cat_1") del_layer = model.get_layer("conv_1") surgeon = Surgeon(model, copy=True) surgeon.add_job("delete_channels", del_layer, channels=channel_index) new_model = surgeon.operate() new_w = new_model.get_layer("dense_1").get_weights() # Calculate next layer's correct weights flat_sz = np.prod(layer.get_output_shape_at(0)[1:]) channel_count = getattr(del_layer, utils.get_channels_attr(del_layer)) channel_index = [i % channel_count for i in channel_index] if data_format == "channels_first": delete_indices = [ x * flat_sz // 2 // channel_count + i for x in channel_index for i in range( 0, flat_sz // 2 // channel_count, ) ] elif data_format == "channels_last": delete_indices = [ x + i for i in range(0, flat_sz, channel_count * 2) for x in channel_index ] else: raise ValueError correct_w = model.get_layer("dense_1").get_weights() correct_w[0] = np.delete(correct_w[0], delete_indices, axis=0) assert weights_equal(correct_w, new_w)
def augment_for_ocv(model): from kerassurgeon import Surgeon # pip install kerassurgeon from tensorflow.keras.layers import Reshape surgeon = Surgeon(model) for layer in model.layers: if 'dropout' in layer.name or 'lambda' in layer.name: surgeon.add_job('delete_layer', layer) if 'flatten' in layer.name: new_layer = Reshape(layer.output_shape[1:]) surgeon.add_job('replace_layer', layer, new_layer=new_layer) new_model = surgeon.operate() return new_model
def prune_model(model, pruning_percent_step, pruning_standart_deviation_part): idxs = [] count = len(model.layers) for layer_idx in range(count): name = model.layers[layer_idx].name if (name.startswith("conv")): w1 = model.layers[layer_idx].get_weights()[0] weight = w1 weights_dict = {} num_filters = len(weight[0, 0, 0, :]) idxs.append(layer_idx) idxs.reverse() surgeon = Surgeon(model) for layer_idx in idxs: layer = model.layers[layer_idx] w1 = model.layers[layer_idx].get_weights()[0] weight = w1 weights_dict = {} num_filters = len(weight[0, 0, 0, :]) # print(num_filters) delete_part_int = (int)(num_filters * pruning_percent_step) + 1 l1_s = [] for j in range(num_filters): l1 = np.sum(abs(weight[:, :, :, j])) filt = f"{j}" weights_dict[filt] = l1 l1_s.append(l1) weights_dict_sort = sorted(weights_dict.items(), key=lambda kv: kv[1]) mean = np.mean(l1_s) std = np.std(l1_s) threshold_l1 = mean - std * pruning_standart_deviation_part delete_idxs = [] for i in range(delete_part_int): if (weights_dict_sort[i][1] > threshold_l1): break delete_idxs.append((int)(weights_dict_sort[i][0])) surgeon.add_job( job="delete_channels", layer=layer, channels=delete_idxs, ) pruned_model = surgeon.operate() return pruned_model
def prune_model(model, apoz_df, n_channels_delete): sorted_apoz_df = apoz_df.sort_values('apoz', ascending=False) high_apoz_index = sorted_apoz_df.iloc[0:n_channels_delete, :] # Create the Surgeon and add a 'delete_channels' job for each layer # whose channels are to be deleted. surgeon = Surgeon(model, copy=True) for name in high_apoz_index.index.unique().values: channels = list( pd.Series(high_apoz_index.loc[name, 'index'], dtype=np.int64).values) surgeon.add_job('delete_channels', model.get_layer(name), channels=channels) return surgeon.operate()
def prune_model(model, apoz_df, n_channels_delete): # Identify 5% of channels with the highest APoZ in model sorted_apoz_df = apoz_df.sort_values("apoz", ascending=False) high_apoz_index = sorted_apoz_df.iloc[0:n_channels_delete, :] # Create the Surgeon and add a 'delete_channels' job for each layer # whose channels are to be deleted. surgeon = Surgeon(model, copy=True) for name in high_apoz_index.index.unique().values: channels = list( pd.Series(high_apoz_index.loc[name, "index"], dtype=np.int64).values) surgeon.add_job("delete_channels", model.get_layer(name), channels=channels) # Delete channels return surgeon.operate()
def pruning_method_fc(model, layer_to_prune, pruning_amount, method): if method == 'L1norm': # Load surgeon package surgeon = Surgeon(model) # Store weights from conv layers [0][1] = [weight][bias] Hence, [0] to store weights only fc_layer_weights = [ model.layers[i].get_weights()[0] for i in layer_to_prune ] for i in range(len(fc_layer_weights)): weight = fc_layer_weights[i] num_filters = weight.shape[1] print('total number of filter: ' + str(num_filters)) filter_to_prune = {} pruning_channel_num = pruning_amount # compute L1-norm of each filter weight and store it in a dictionary(filter_to_prune) for j in range(num_filters): L1_norm = np.sum(abs(weight[:, j])) filter_number = 'filter_{}'.format(j) filter_to_prune[filter_number] = L1_norm # sort the filter according to the ascending L1 value # pruning[0]: sort by name, pruning[1]: sort by value filter_to_prune_sort = sorted(filter_to_prune.items(), key=lambda pruning: pruning[1]) print(filter_to_prune_sort) # extracting filter number from '(filter_2, 0.515..), eg) extracting '2' from '(filter_2, 0.515..) remove_channel = [ int(filter_to_prune_sort[i][0].split("_")[1]) for i in range(0, pruning_channel_num[i]) ] print(remove_channel) # delete filters with lowest L1 norm values surgeon.add_job('delete_channels', model.layers[layer_to_prune[i]], channels=remove_channel) model_pruned = surgeon.operate() return model_pruned
def prune_multiple_layers(model, pruned_matrix, opt): conv_indexes = [i for i, v in enumerate(model.layers) if 'conv' in v.name] layers_to_prune = np.unique(pruned_matrix[:, 0]) surgeon = Surgeon(model, copy=True) to_prune = pruned_matrix to_prune[:, 0] = np.array([conv_indexes[i] for i in to_prune[:, 0]]) layers_to_prune = np.unique(to_prune[:, 0]) for layer_ix in layers_to_prune: pruned_filters = [x[1] for x in to_prune if x[0] == layer_ix] pruned_layer = model.layers[layer_ix] surgeon.add_job('delete_channels', pruned_layer, channels=pruned_filters) model_pruned = surgeon.operate() model_pruned.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy']) return model_pruned
def percentage_pruning(self, percentage): from kerassurgeon import Surgeon # k surgeon = Surgeon(self.model) layer_list = self.model.layers # list of layers in the model to_prune_list = [] for i in range(len(layer_list)): if 'conv' in str(layer_list[i]): to_prune_list.append( i ) # check the indexes of the convolutional layers in the model print(to_prune_list) n_filters = [] for i in to_prune_list: # gets the number of filter in each convolutional layer n_filters.append( self.model.get_layer(index=i).get_config()['filters']) mega_list = [] print(n_filters) for kk, _ in enumerate( to_prune_list): # iterate on each convolutional layer num_filter_layer = n_filters[ kk] # number of filters in the current layer, which is to_prune_list[kk] random_index = [] # index of the filters to be pruned for jj in range(int(np.ceil( percentage[kk] * num_filter_layer))): # P% of the filters tmp = np.random.randint(0, num_filter_layer) while tmp in random_index: tmp = np.random.randint( 0, num_filter_layer ) # if the filter has alread benn choosen, try again random_index.append(tmp) mega_list.append( random_index ) # contain a list of all the random index of that layer, the length of the mega list is thus the length # of the convolutional filters layerr = self.model.layers[to_prune_list[kk]] surgeon.add_job('delete_channels', layerr, channels=random_index) self.model = surgeon.operate() return mega_list
def test_delete_all_channels_in_branch(): input_1 = Input(shape=(20, 20, 3)) conv_1 = Conv2D(2, [3, 3], name='conv_1') conv_2 = Conv2D(3, [3, 3], name='conv_2') cat_1 = Concatenate(name='cat_1') x = conv_1(input_1) y = conv_2(input_1) output_1 = cat_1([x, y]) model_1 = utils.clean_copy(Model(input_1, output_1)) surgeon = Surgeon(model_1, copy=True) surgeon.add_job('delete_channels', model_1.get_layer('conv_1'), channels=[0, 1]) model_2 = surgeon.operate() output_1 = conv_2(input_1) model_2_exp = utils.clean_copy(Model(input_1, output_1)) config_1 = model_2.get_config() config_2 = model_2_exp.get_config() config_2['name'] = config_1['name'] # make the config names identical assert json.dumps(config_1) == json.dumps(config_2)
def prune_model_random(model, percent_channels_delete): start = None end = None # Create the Surgeon and add a 'delete_channels' job for each layer # whose channels are to be deleted. surgeon = Surgeon(model, copy=True) for layer in model.layers[start:end]: if identify.is_conv(layer): print(layer.name) num_total_channels = layer.output_shape[-1] total_channels = list(range(num_total_channels)) shuffle(total_channels) num_removed_channels = int( num_total_channels * percent_channels_delete / 100) // 16 * 16 if num_removed_channels > 0: channels = total_channels[:num_removed_channels] print('before add_job:', layer.name, channels, len(channels), num_total_channels) surgeon.add_job('delete_channels', layer, channels=channels) # Delete channels return surgeon.operate()
def prune(model, thresh=0.1): ''' model: model to prune thresh: neurons, corresponding to BSFilter layers lower than thresh will be removed ''' bs_idx = [] for idx, l in enumerate(model.layers): if l.__class__.__name__ == "BSFilter": bs_idx.append(idx) weights = [] surgeon = Surgeon(model) for i in bs_idx: weights = model.layers[i].get_weights()[0] l = model.layers[i] to_prune = model.layers[i - 1] surgeon.add_job('delete_layer', l) surgeon.add_job('delete_channels', to_prune, channels=np.where(weights < thresh)[0].tolist()) return surgeon.operate()
def Prun_channel_1(self, layer, layer_1, config, channels_p): weight = layer.weight op_type = layer.type op_name = layer.name assert 0 <= config.get('sparsity') < 1 assert op_type in ['Conv1D', 'Conv2D'] assert op_type in config['op_types'] # op_name = layer.name # assert 0 <= config.get('sparsity') < 1 # assert op_type in ['Conv1D', 'Conv2D'] # assert op_type in config['op_types'] #if layer.name in self.epoch_pruned_layers: # assert layer.name in self.mask_dict # return self.mask_dict.get(layer.name) try: w = tf.stop_gradient(tf.transpose(tf.reshape(weight, (-1, weight.shape[-1])), [1, 0])) masks = np.ones(w.shape) num_filters = w.shape[0] num_prune = int(num_filters * config.get('sparsity')) if num_filters < 2 or num_prune < 1: return masks #min_gm_idx = self._get_min_gm_kernel_idx_m(w, num_prune) surgeon = Surgeon(self.bound_model, copy=False) channels = channels_p surgeon.add_job('delete_channels', layer_1, channels=channels) #for idx in min_gm_idx: # masks[idx] = 0. finally: masks = tf.reshape(tf.transpose(masks, [1, 0]), weight.shape) masks = tf.Variable(masks) self.mask_dict.update({op_name: masks}) self.epoch_pruned_layers.add(layer.name) return surgeon.operate()
def random_pruning(self, P): from kerassurgeon import Surgeon surgeon = Surgeon(self.model) layer_list = self.model.layers # list of layers in the model to_prune_list = [] for i in range(len(layer_list)): if 'conv' in str(layer_list[i]): #select convolutional layers only to_prune_list.append(i) # gets their indexes print(to_prune_list) n_filters = [] for i in to_prune_list: # gets the number of convolutional filters in every layer n_filters.append( self.model.get_layer(index=i).get_config()['filters']) mega_list = [ ] #mega list is a list of list where every element are the index of the filter removed in that layer print(n_filters) for kk in range(len(to_prune_list)): # for every conv layer num_filter_layer = n_filters[kk] # number of filters in that layer random_index = [] # index of the filters to be pruned for jj in range(int(np.ceil( P * num_filter_layer))): # gets random indexes tmp = np.random.randint(0, num_filter_layer) while tmp in random_index: tmp = np.random.randint( 0, num_filter_layer ) # if the filter has alread benn choosen, try again random_index.append(tmp) mega_list.append(random_index) layerr = self.model.layers[to_prune_list[kk]] surgeon.add_job('delete_channels', layerr, channels=random_index) self.model = surgeon.operate() return mega_list
def prune_model_by_layer(model, percent_channels_delete, image_dir, image_lists, preprocess_fct): start = None end = None #model.summary() # Create the Surgeon and add a 'delete_channels' job for each layer # whose channels are to be deleted. surgeon = Surgeon(model, copy=True) for layer in model.layers[start:end]: if identify.is_conv(layer): print(layer.name) num_total_channels = layer.output_shape[-1] num_removed_channels = int( num_total_channels * percent_channels_delete / 100) // 16 * 16 if num_removed_channels > 0: if False: images, y = get_images(image_dir, image_lists, preprocess_fct) #channels_importance=get_channels_apoz_importance(model, layer, images) #channels_importance=get_channels_importance(model, layer, images,y) #channels_importance=get_channels_loss(model, layer, images,y) #channels_importance=get_channels_gradients(model, layer, images,y) #channels_importance=get_channels_importance_with_gradient(model, layer, images,y) else: channels_importance = get_channels_l1_norm(model, layer) total_channels_sorted = np.argsort(channels_importance) #print('channels_importance:',channels_importance.shape,total_channels_sorted[:5]) channels = total_channels_sorted[:num_removed_channels] print('before add_job:', layer.name, channels, channels_importance[channels], len(channels), num_total_channels) surgeon.add_job('delete_channels', layer, channels=channels) # Delete channels return surgeon.operate()
def pruning(self, rank, vec, metodo, x_, y, classi=[]): lunghezza = len(rank) if type( vec ) == list: # vec can contain either the percentage of filters to remove or their exact number if type(vec[0]) == int: type_flag = 0 elif type(vec[0]) == float: type_flag = 1 else: type_flag = 2 if lunghezza == 3: # cm_c criterion is used for the rank mega_norm0 = rank[0] mega_norm1 = rank[1] mega_norm2 = rank[2] n_filters = [ 64, 64, 128, 128, 256, 256, 256, 512, 512, 512, 512, 512, 512 ] # number of filters in vgg 16 un_arr0 = [oo for o in mega_norm0 for oo in o] un_arr1 = [oo for o in mega_norm1 for oo in o] un_arr2 = [oo for o in mega_norm2 for oo in o] if metodo == 0: # rank is used layer by layer with no training in between surgeon = Surgeon(self.model) B = classi[0] M = classi[1] layer = [ lay for lay in self.model.layers if 'conv' in lay.name ] for idx, i in enumerate( layer ): #get the best and worst filters for every class. best0 = list( np.argsort( mega_norm0[idx])[-int(B * len(mega_norm0[idx]))::]) best1 = list( np.argsort( mega_norm1[idx])[-int(B * len(mega_norm0[idx]))::]) best2 = list( np.argsort( mega_norm2[idx])[-int(B * len(mega_norm0[idx]))::]) worst0 = list( np.argsort( mega_norm0[idx])[0:int(M * len(mega_norm0[idx]))]) worst1 = list( np.argsort( mega_norm1[idx])[0:int(M * len(mega_norm0[idx]))]) worst2 = list( np.argsort( mega_norm2[idx])[0:int(M * len(mega_norm0[idx]))]) to_prune = [] for iii in worst0: if (iii not in best1) and (iii not in best2): if iii not in to_prune: to_prune.append(iii) for iii in worst1: if (iii not in best0) and (iii not in best2): if iii not in to_prune: to_prune.append(iii) for iii in worst2: if (iii not in best0) and (iii not in best1): if iii not in to_prune: to_prune.append(iii) print('number of filters to remove=', len(to_prune)) surgeon.add_job('delete_channels', layer[idx], channels=to_prune) self.model = surgeon.operate() elif metodo == 1: # rank is used globally with no training in between surgeon = Surgeon(self.model) B = classi[0] M = classi[1] best0 = list( np.argsort(un_arr0)[-int(B * sum(self.check_nfilters()))::]) best1 = list( np.argsort(un_arr1)[-int(B * sum(self.check_nfilters()))::]) best2 = list( np.argsort(un_arr2)[-int(B * sum(self.check_nfilters()))::]) worst0 = list( np.argsort(un_arr0)[0:int(M * sum(self.check_nfilters()))]) worst1 = list( np.argsort(un_arr1)[0:int(M * sum(self.check_nfilters()))]) worst2 = list( np.argsort(un_arr2)[0:int(M * sum(self.check_nfilters()))]) lay = [i for i in self.model.layers if 'conv' in i.name] for indice, hj in enumerate(lay): to_prune = [] for iii in worst0: if (iii not in best1) and (iii not in best2): if un_arr0[iii] in mega_norm0[indice]: tmp = np.argwhere( mega_norm0[indice] == un_arr0[iii]) for gt in tmp: if gt not in to_prune and len( to_prune) < int( n_filters[indice] * 0.95): # used to prevent a whole network to be pruned to_prune.append(int(gt)) for iii in worst1: if (iii not in best0) and (iii not in best2): if un_arr1[iii] in mega_norm1[indice]: tmp = np.argwhere( mega_norm1[indice] == un_arr1[iii]) for gt in tmp: if gt not in to_prune and len( to_prune) < int( n_filters[indice] * 0.95): to_prune.append(int(gt)) for iii in worst2: if (iii not in best0) and (iii not in best1): if un_arr2[iii] in mega_norm2[indice]: tmp = np.argwhere( mega_norm2[indice] == un_arr2[iii]) for gt in tmp: if gt not in to_prune and len( to_prune) < int( n_filters[indice] * 0.95): to_prune.append(int(gt)) surgeon.add_job('delete_channels', hj, channels=to_prune) print('number of filters to remove=', len(to_prune)) self.model = surgeon.operate() elif metodo == 2: # rank is used layer by layer but there is a training for every pruning step surgeon = Surgeon(self.model) B = classi[0] M = classi[1] save_path = os.path.join(os.getcwd(), 'saved_model_tmp.h5') self.model.save(save_path) # save for idx in range(12, -1, -1): self.model = load_model(save_path) trainable_lay = [0 for i in range(16) ] # all layers are trainable trainable_lay[idx] = 1 self.set_trainable_layers(trainable_lay) surgeon = Surgeon(self.model) layer = [ jj for jj in self.model.layers if 'conv' in jj.name ] best0 = list( np.argsort( mega_norm0[idx])[-int(B * len(mega_norm0[idx]))::]) best1 = list( np.argsort( mega_norm1[idx])[-int(B * len(mega_norm0[idx]))::]) best2 = list( np.argsort( mega_norm2[idx])[-int(B * len(mega_norm0[idx]))::]) worst0 = list( np.argsort( mega_norm0[idx])[0:int(M * len(mega_norm0[idx]))]) worst1 = list( np.argsort( mega_norm1[idx])[0:int(M * len(mega_norm0[idx]))]) worst2 = list( np.argsort( mega_norm2[idx])[0:int(M * len(mega_norm0[idx]))]) to_prune = [] for iii in worst0: if (iii not in best1) and (iii not in best2): if iii not in to_prune: to_prune.append(iii) for iii in worst1: if (iii not in best0) and (iii not in best2): if iii not in to_prune: to_prune.append(iii) for iii in worst2: if (iii not in best0) and (iii not in best1): if iii not in to_prune: to_prune.append(iii) print('number of filters to remove=', len(to_prune)) surgeon.add_job('delete_channels', layer[idx], channels=to_prune) self.model = surgeon.operate() sgd = optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True) self.model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) for i in range(5): # self.model.fit(x_, y, epochs=3, batch_size=8) performance = self.model.evaluate(x_, y) if performance[1] > 0.9: break self.model.save(save_path) del self.model K.clear_session() tf.reset_default_graph() self.model = load_model(save_path) elif lunghezza != 3: # l1 and apz criteria if metodo == 0: surgeon = Surgeon(self.model) layer = [jj for jj in self.model.layers if 'conv' in jj.name] un_arr0 = [oo for o in rank for oo in o] for idx in range(13): norm = rank[idx] if type_flag == 0: indici = np.array(norm).argsort()[0:vec[idx]] elif type_flag == 1: indici = np.array( norm).argsort()[0:int(vec[idx] * len(rank[idx]))] to_remove_channel = indici.tolist() surgeon.add_job('delete_channels', layer[idx], channels=indici) self.model = surgeon.operate() elif metodo == 1: surgeon = Surgeon(self.model) un_arr = [oo for o in rank for oo in o] layer = [jj for jj in self.model.layers if 'conv' in jj.name] if type_flag == 0: norm = np.argsort(un_arr)[0:sum(vec)] # ordered vector elif type_flag == 1: # percentage is used norm = np.argsort( un_arr)[0:int(np.sum(self.check_nfilters()) * vec[0])] print('len norm=', len(norm)) for idx, jj in enumerate(layer): to_prune = [] for kk in norm: if un_arr[kk] in rank[idx]: tmp = np.argwhere( rank[idx] == un_arr[kk]).squeeze() try: if int(tmp) not in to_prune and len( to_prune) < int(n_filters[idx] * 0.95): to_prune.append(int(tmp)) except: for ij in tmp: if ij not in to_prune and len( to_prune) < int( n_filters[idx] * 0.95): to_prune.append(ij) print('len(to_prune)', len(to_prune), idx) surgeon.add_job('delete_channels', layer[idx], channels=to_prune) self.model = surgeon.operate() elif metodo == 2: # faccio un train per ogni strato surgeon = Surgeon(self.model) self.model.save('R:\\saved_model_tmp.h5') for idx in range(12, -1, -1): self.model = load_model('R:\\saved_model_tmp.h5') # trainable_lay=[0 for i in range(16)] # trainable_lay[idx]=1 # self.set_trainable_layers(trainable_lay) surgeon = Surgeon(self.model) layer = [ jj for jj in self.model.layers if 'conv' in jj.name ] norm = rank[idx] if type_flag == 0: #lavoro direttamente con il numero di filtri indici = np.array(norm).argsort()[0:vec[idx]] print('len(indici)=', len(indici)) elif type_flag == 1: # specifico solo la percentuale indici = np.array( norm).argsort()[0:int(vec[idx] * len(rank[idx]))] print('len(indici)=', len(indici)) elif type_flag == 2: #specifico la distanza da una sigma std = np.std(norm) index = (norm - np.man(norm)) < -std indici = np.argwhere(index == 1) print('number of filters to remove=', len(indici)) surgeon.add_job('delete_channels', layer[idx], channels=indici) self.model = surgeon.operate() sgd = optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True) self.model.compile(loss='categorical_crossentropy', optimizer=sgd, metrics=['accuracy']) for i in range(5): self.model.fit(x_, y, epochs=3, batch_size=8) performance = self.model.evaluate(x_, y) if performance[1] > 0.9: break save_path = os.path.join(os.getcwd(), 'saved_model_tmp.h5') self.model.save(save_path) del self.model K.clear_session() tf.reset_default_graph() self.model = load_model(save_path)
def pruning_method_conv(model, layer_to_prune, pruning_amount, method): if method == 'L1norm': # Load surgeon package surgeon = Surgeon(model) # Store weights from conv layers [0][1] = [weight][bias] Hence, [0] to store weights only conv_layer_weights = [ model.layers[i].get_weights()[0] for i in layer_to_prune ] print('number of layer to prune: ' + str(len(conv_layer_weights))) for i in range(len(conv_layer_weights)): if pruning_amount[i] == 0: continue weight = conv_layer_weights[i] num_filters = len(weight[0, 0, 0, :]) print('total number of filter: ' + str(num_filters)) weight_removable = {} # compute L1-nom of each filter weight and store it in a dictionary(weight_removable) for j in range(num_filters): L1_norm = np.sum(abs(weight[:, :, :, j])) filter_number = 'filter_{}'.format(j) weight_removable[filter_number] = L1_norm # sort the filter according to the ascending L1 value weight_removable_sort = sorted(weight_removable.items(), key=lambda kv: kv[1]) # 'filter_24'(string) -> 24(int), # extracting filter number from '(filter_2, 0.515..), eg) extracting '2' from '(filter_2, 0.515..) remove_channel = [ int(weight_removable_sort[i][0].split("_")[1]) for i in range(0, pruning_amount[i]) ] # delete filters with lowest scores surgeon.add_job('delete_channels', model.layers[layer_to_prune[i]], channels=remove_channel) model_pruned = surgeon.operate() return model_pruned elif method == 'geometric_median_conv': # Load surgeon package surgeon = Surgeon(model) # Store weights from conv layers [0][1] = [weight][bias] Hence, [0] to store weights only conv_layer_weights = [ model.layers[i].get_weights()[0] for i in layer_to_prune ] print('number of layer to prune: ' + str(len(conv_layer_weights))) for i in range(len(conv_layer_weights)): if pruning_amount[i] == 0: continue weight = conv_layer_weights[i] num_filters = len(weight[0, 0, 0, :]) print('total number of filter: ' + str(num_filters)) weight_removable = {} # 1. Reduce dimension 4D -> 2D # 2. Normalization (L1, L2) # 3. Sort norm value => get index order # 4. Sort weight by index order # 5. Calculate distance between coordinates # 6. Sum distance (of each filter) norm_val = geometric_median(weight, "euclidean", "L1") # compute L1-nom of each filter weight and store it in a dictionary(weight_removable) for j in range(num_filters): filter_number = 'filter_{}'.format(j) weight_removable[filter_number] = norm_val[j] # sort the filter according to the ascending L1 value weight_removable_sort = sorted(weight_removable.items(), key=lambda kv: kv[1]) # print(weight_removable_sort) # 'filter_24'(string) -> 24(int), # extracting filter number from '(filter_2, 0.515..), eg) extracting '2' from '(filter_2, 0.515..) remove_channel = [ int(weight_removable_sort[i][0].split("_")[1]) for i in range(0, pruning_amount[i]) ] # print(remove_channel) # delete filters with lowest scores surgeon.add_job('delete_channels', model.layers[layer_to_prune[i]], channels=remove_channel) model_pruned = surgeon.operate() return model_pruned