def layer_test_helper_flatten_1d(layer, channel_index):
# This should test that the output is the correct shape so it should pass
# into a Dense layer rather than a Conv layer.
# The weighted layer is the previous layer,
# Create model
main_input = Input(shape=list(random.randint(10, 20, size=2)))
x = Conv1D(3, 3)(main_input)
x = layer(x)
x = Flatten()(x)
main_output = Dense(5)(x)
model = Model(inputs=main_input, outputs=main_output)
# Delete channels
del_layer_index = 1
next_layer_index = 4
del_layer = model.layers[del_layer_index]
surgeon = Surgeon(model)
surgeon.add_job("delete_channels", del_layer, channels=channel_index)
new_model = surgeon.operate()
new_w = new_model.layers[next_layer_index].get_weights()
# Calculate next layer's correct weights
flat_sz = np.prod(layer.get_output_shape_at(0)[1:])
channel_count = getattr(del_layer, utils.get_channels_attr(del_layer))
channel_index = [i % channel_count for i in channel_index]
delete_indices = [
x + i for i in range(0, flat_sz, channel_count) for x in channel_index
]
correct_w = model.layers[next_layer_index].get_weights()
correct_w[0] = np.delete(correct_w[0], delete_indices, axis=0)
assert weights_equal(correct_w, new_w)
def test_delete_channels_downstream_sharing():
# Create all model layers
input_1 = Input(shape=(5,))
dense_1 = Dense(4, name='dense_1')
dense_2 = Dense(4, name='dense_2')
dense_3 = Dense(3, name='dense_3')
# Create the base model
x = dense_1(input_1)
y = dense_2(input_1)
output_1 = dense_3(x)
output_2 = dense_3(y)
model_1 = utils.clean_copy(Model(input_1, [output_1, output_2]))
# Delete channels from dense_1 and dense_2
surgeon = Surgeon(model_1)
surgeon.add_job('delete_channels', model_1.get_layer(dense_1.name), channels=[0])
surgeon.add_job('delete_channels', model_1.get_layer(dense_2.name), channels=[1])
model_2 = surgeon.operate()
# Create the expected model
# input_1 = Input(shape=(5,))
dense_1_exp = Dense(3, name='dense_1')
dense_2_exp = Dense(3, name='dense_2')
dense_3_exp = Dense(3, name='dense_3')
# Create the base model
x = dense_1_exp(input_1)
y = dense_2_exp(input_1)
output_1 = dense_3_exp(x)
output_2 = dense_3_exp(y)
model_2_exp = utils.clean_copy(Model(input_1, [output_1, output_2]))
config_1 = model_2.get_config()
config_2 = model_2_exp.get_config()
config_2['name'] = config_1['name'] # make the config names identical
assert json.dumps(config_1) == json.dumps(config_2)
def test_delete_all_channels_in_long_branch():
input_1 = Input(shape=(20, 20, 3))
conv_1 = Conv2D(2, [3, 3], name="conv_1")
conv_2 = Conv2D(3, [3, 3], name="conv_2")
conv_3 = Conv2D(4, [1, 1], name="conv_3")
cat_1 = Concatenate(name="cat_1")
x = conv_1(input_1)
x = conv_3(x)
y = conv_2(input_1)
output_1 = cat_1([x, y])
model_1 = utils.clean_copy(Model(input_1, output_1))
surgeon = Surgeon(model_1, copy=True)
surgeon.add_job("delete_channels",
model_1.get_layer("conv_1"),
channels=[0, 1])
model_2 = surgeon.operate()
output_1 = conv_2(input_1)
model_2_exp = utils.clean_copy(Model(input_1, output_1))
config_1 = model_2.get_config()
config_2 = model_2_exp.get_config()
config_2["name"] = config_1["name"] # make the config names identical
assert config_1 == config_2
def mask_pruning(self, Vec):
vec = []
for idx, i in enumerate(Vec):
if i == 1:
vec.append(
idx
) # appendo nel vettore vec gli indici dei filtri da eliminare
#this function takes a vector of indexes, if there is a 1 in the j-th position the j-th filter of the network will be pruned.
from kerassurgeon import Surgeon # k
surgeon = Surgeon(self.model)
layer_list = self.model.layers # list of layers in the model e.g [<keras.engine.input_layer.InputLayer at 0x2068f6c2048>,...
to_prune_list = []
for i in range(len(layer_list)):
if 'conv' in str(layer_list[i]):
to_prune_list.append(
i
) # check the indexes of the convolutional layers in the model
n_filters = []
# print(to_prune_list)
for i in to_prune_list:
n_filters.append(
self.model.get_layer(index=i).get_config()['filters']
) # gets the number of filter in each convolutional layer
# print('num,er of filters=',n_filters)
# return n_filters
sumlist = [0 for i in n_filters
] # vettore lungo quanto il numero di strati
for idx in range(len(n_filters)):
sumlist[idx] = sum(n_filters[0:idx +
1]) #[64,64,128]->[64,128,256]
# print(sumlist)
# print('vec=',vec)
mega_list = [[], [], [], [], [], [], [], [], [], [], [], [], []]
for num in vec:
for idx, i in enumerate(sumlist):
if num - i < 0:
strato = idx
if idx == 0:
numero = num
else:
numero = num - sumlist[idx - 1]
# print('strato=',idx,'numero=',numero)
mega_list[idx].append(numero)
break
# print('dimensione di mega list=',len(mega_list))
for kk in range(
len(to_prune_list)): # iterate on each convolutional layer
# of the convolutional filters
layerr = self.model.layers[to_prune_list[kk]]
# print('len di mega_list[kk]=',len(mega_list[kk]))
surgeon.add_job('delete_channels', layerr, channels=mega_list[kk])
self.model = surgeon.operate()
return 0
def semi_random_pruning(self,P,kk):
from kerassurgeon import Surgeon
layer_list=self.model.layers # list of layers in the model e.g [<keras.engine.input_layer.InputLayer at 0x2068f6c2048>,...
to_prune_list=[]
for i in range(len(layer_list)):
if 'Conv2D' in str(layer_list[i]):
to_prune_list.append(i) # check the indexes of the convolutional layers in the model
n_filters=[]
# print(to_prune_list)
for i in to_prune_list:
n_filters.append(self.model.get_layer(index=i).get_config()['filters']) # gets the number of filter in each convolutional layer
# print('num,er of filters=',n_filters)
# return n_filters
mega_list=[]
# for kk in range(len(to_prune_list)): # iterate on each convolutional layer
num_filter_layer=n_filters[kk]# number of filters in the current layer, which is to_prune_list[kk]
random_index=[] # index of the filters to be pruned
for jj in range(int(np.ceil(P*num_filter_layer))): # P% of the filters
tmp=np.random.randint(0,num_filter_layer)
while tmp in random_index:
tmp=np.random.randint(0,num_filter_layer) # if the filter has alread benn choosen, try again
random_index.append(tmp)
# mega_list.append(random_index) # contain a list of all the random index of that layer, the length of the mega list is thus the length
# of the convolutional filters
layerr=self.model.layers[to_prune_list[kk]]
print(str(layerr))
surgeon = Surgeon(self.model)
surgeon.add_job('delete_channels', layerr, channels=random_index)
self.model = surgeon.operate()
def test_delete_channels_merge_concatenate(channel_index, data_format):
# This should test that the output is the correct shape so it should pass
# into a Dense layer rather than a Conv layer.
# The weighted layer is the previous layer,
# Create model
if data_format == "channels_first":
axis = 1
elif data_format == "channels_last":
axis = -1
else:
raise ValueError
input_shape = list(random.randint(10, 20, size=3))
input_1 = Input(shape=input_shape)
input_2 = Input(shape=input_shape)
x = Conv2D(3, [3, 3], data_format=data_format, name="conv_1")(input_1)
y = Conv2D(3, [3, 3], data_format=data_format, name="conv_2")(input_2)
x = Concatenate(axis=axis, name="cat_1")([x, y])
x = Flatten()(x)
main_output = Dense(5, name="dense_1")(x)
model = Model(inputs=[input_1, input_2], outputs=main_output)
old_w = model.get_layer("dense_1").get_weights()
# Delete channels
layer = model.get_layer("cat_1")
del_layer = model.get_layer("conv_1")
surgeon = Surgeon(model, copy=True)
surgeon.add_job("delete_channels", del_layer, channels=channel_index)
new_model = surgeon.operate()
new_w = new_model.get_layer("dense_1").get_weights()
# Calculate next layer's correct weights
flat_sz = np.prod(layer.get_output_shape_at(0)[1:])
channel_count = getattr(del_layer, utils.get_channels_attr(del_layer))
channel_index = [i % channel_count for i in channel_index]
if data_format == "channels_first":
delete_indices = [
x * flat_sz // 2 // channel_count + i for x in channel_index
for i in range(
0,
flat_sz // 2 // channel_count,
)
]
elif data_format == "channels_last":
delete_indices = [
x + i for i in range(0, flat_sz, channel_count * 2)
for x in channel_index
]
else:
raise ValueError
correct_w = model.get_layer("dense_1").get_weights()
correct_w[0] = np.delete(correct_w[0], delete_indices, axis=0)
assert weights_equal(correct_w, new_w)
def augment_for_ocv(model):
from kerassurgeon import Surgeon # pip install kerassurgeon
from tensorflow.keras.layers import Reshape
surgeon = Surgeon(model)
for layer in model.layers:
if 'dropout' in layer.name or 'lambda' in layer.name:
surgeon.add_job('delete_layer', layer)
if 'flatten' in layer.name:
new_layer = Reshape(layer.output_shape[1:])
surgeon.add_job('replace_layer', layer, new_layer=new_layer)
new_model = surgeon.operate()
return new_model
def prune_model(model, pruning_percent_step, pruning_standart_deviation_part):
idxs = []
count = len(model.layers)
for layer_idx in range(count):
name = model.layers[layer_idx].name
if (name.startswith("conv")):
w1 = model.layers[layer_idx].get_weights()[0]
weight = w1
weights_dict = {}
num_filters = len(weight[0, 0, 0, :])
idxs.append(layer_idx)
idxs.reverse()
surgeon = Surgeon(model)
for layer_idx in idxs:
layer = model.layers[layer_idx]
w1 = model.layers[layer_idx].get_weights()[0]
weight = w1
weights_dict = {}
num_filters = len(weight[0, 0, 0, :])
# print(num_filters)
delete_part_int = (int)(num_filters * pruning_percent_step) + 1
l1_s = []
for j in range(num_filters):
l1 = np.sum(abs(weight[:, :, :, j]))
filt = f"{j}"
weights_dict[filt] = l1
l1_s.append(l1)
weights_dict_sort = sorted(weights_dict.items(), key=lambda kv: kv[1])
mean = np.mean(l1_s)
std = np.std(l1_s)
threshold_l1 = mean - std * pruning_standart_deviation_part
delete_idxs = []
for i in range(delete_part_int):
if (weights_dict_sort[i][1] > threshold_l1):
break
delete_idxs.append((int)(weights_dict_sort[i][0]))
surgeon.add_job(
job="delete_channels",
layer=layer,
channels=delete_idxs,
)
pruned_model = surgeon.operate()
return pruned_model
def prune_model(model, apoz_df, n_channels_delete):
sorted_apoz_df = apoz_df.sort_values('apoz', ascending=False)
high_apoz_index = sorted_apoz_df.iloc[0:n_channels_delete, :]
# Create the Surgeon and add a 'delete_channels' job for each layer
# whose channels are to be deleted.
surgeon = Surgeon(model, copy=True)
for name in high_apoz_index.index.unique().values:
channels = list(
pd.Series(high_apoz_index.loc[name, 'index'],
dtype=np.int64).values)
surgeon.add_job('delete_channels',
model.get_layer(name),
channels=channels)
return surgeon.operate()
def prune_model(model, apoz_df, n_channels_delete):
# Identify 5% of channels with the highest APoZ in model
sorted_apoz_df = apoz_df.sort_values("apoz", ascending=False)
high_apoz_index = sorted_apoz_df.iloc[0:n_channels_delete, :]
# Create the Surgeon and add a 'delete_channels' job for each layer
# whose channels are to be deleted.
surgeon = Surgeon(model, copy=True)
for name in high_apoz_index.index.unique().values:
channels = list(
pd.Series(high_apoz_index.loc[name, "index"],
dtype=np.int64).values)
surgeon.add_job("delete_channels",
model.get_layer(name),
channels=channels)
# Delete channels
return surgeon.operate()
def pruning_method_fc(model, layer_to_prune, pruning_amount, method):
if method == 'L1norm':
# Load surgeon package
surgeon = Surgeon(model)
# Store weights from conv layers [0][1] = [weight][bias] Hence, [0] to store weights only
fc_layer_weights = [
model.layers[i].get_weights()[0] for i in layer_to_prune
]
for i in range(len(fc_layer_weights)):
weight = fc_layer_weights[i]
num_filters = weight.shape[1]
print('total number of filter: ' + str(num_filters))
filter_to_prune = {}
pruning_channel_num = pruning_amount
# compute L1-norm of each filter weight and store it in a dictionary(filter_to_prune)
for j in range(num_filters):
L1_norm = np.sum(abs(weight[:, j]))
filter_number = 'filter_{}'.format(j)
filter_to_prune[filter_number] = L1_norm
# sort the filter according to the ascending L1 value
# pruning[0]: sort by name, pruning[1]: sort by value
filter_to_prune_sort = sorted(filter_to_prune.items(),
key=lambda pruning: pruning[1])
print(filter_to_prune_sort)
# extracting filter number from '(filter_2, 0.515..), eg) extracting '2' from '(filter_2, 0.515..)
remove_channel = [
int(filter_to_prune_sort[i][0].split("_")[1])
for i in range(0, pruning_channel_num[i])
]
print(remove_channel)
# delete filters with lowest L1 norm values
surgeon.add_job('delete_channels',
model.layers[layer_to_prune[i]],
channels=remove_channel)
model_pruned = surgeon.operate()
return model_pruned
def prune_multiple_layers(model, pruned_matrix, opt):
conv_indexes = [i for i, v in enumerate(model.layers) if 'conv' in v.name]
layers_to_prune = np.unique(pruned_matrix[:, 0])
surgeon = Surgeon(model, copy=True)
to_prune = pruned_matrix
to_prune[:, 0] = np.array([conv_indexes[i] for i in to_prune[:, 0]])
layers_to_prune = np.unique(to_prune[:, 0])
for layer_ix in layers_to_prune:
pruned_filters = [x[1] for x in to_prune if x[0] == layer_ix]
pruned_layer = model.layers[layer_ix]
surgeon.add_job('delete_channels',
pruned_layer,
channels=pruned_filters)
model_pruned = surgeon.operate()
model_pruned.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
return model_pruned
def percentage_pruning(self, percentage):
from kerassurgeon import Surgeon # k
surgeon = Surgeon(self.model)
layer_list = self.model.layers # list of layers in the model
to_prune_list = []
for i in range(len(layer_list)):
if 'conv' in str(layer_list[i]):
to_prune_list.append(
i
) # check the indexes of the convolutional layers in the model
print(to_prune_list)
n_filters = []
for i in to_prune_list:
# gets the number of filter in each convolutional layer
n_filters.append(
self.model.get_layer(index=i).get_config()['filters'])
mega_list = []
print(n_filters)
for kk, _ in enumerate(
to_prune_list): # iterate on each convolutional layer
num_filter_layer = n_filters[
kk] # number of filters in the current layer, which is to_prune_list[kk]
random_index = [] # index of the filters to be pruned
for jj in range(int(np.ceil(
percentage[kk] * num_filter_layer))): # P% of the filters
tmp = np.random.randint(0, num_filter_layer)
while tmp in random_index:
tmp = np.random.randint(
0, num_filter_layer
) # if the filter has alread benn choosen, try again
random_index.append(tmp)
mega_list.append(
random_index
) # contain a list of all the random index of that layer, the length of the mega list is thus the length
# of the convolutional filters
layerr = self.model.layers[to_prune_list[kk]]
surgeon.add_job('delete_channels', layerr, channels=random_index)
self.model = surgeon.operate()
return mega_list
def test_delete_all_channels_in_branch():
input_1 = Input(shape=(20, 20, 3))
conv_1 = Conv2D(2, [3, 3], name='conv_1')
conv_2 = Conv2D(3, [3, 3], name='conv_2')
cat_1 = Concatenate(name='cat_1')
x = conv_1(input_1)
y = conv_2(input_1)
output_1 = cat_1([x, y])
model_1 = utils.clean_copy(Model(input_1, output_1))
surgeon = Surgeon(model_1, copy=True)
surgeon.add_job('delete_channels', model_1.get_layer('conv_1'), channels=[0, 1])
model_2 = surgeon.operate()
output_1 = conv_2(input_1)
model_2_exp = utils.clean_copy(Model(input_1, output_1))
config_1 = model_2.get_config()
config_2 = model_2_exp.get_config()
config_2['name'] = config_1['name'] # make the config names identical
assert json.dumps(config_1) == json.dumps(config_2)
def prune_model_random(model, percent_channels_delete):
start = None
end = None
# Create the Surgeon and add a 'delete_channels' job for each layer
# whose channels are to be deleted.
surgeon = Surgeon(model, copy=True)
for layer in model.layers[start:end]:
if identify.is_conv(layer):
print(layer.name)
num_total_channels = layer.output_shape[-1]
total_channels = list(range(num_total_channels))
shuffle(total_channels)
num_removed_channels = int(
num_total_channels * percent_channels_delete / 100) // 16 * 16
if num_removed_channels > 0:
channels = total_channels[:num_removed_channels]
print('before add_job:', layer.name, channels, len(channels),
num_total_channels)
surgeon.add_job('delete_channels', layer, channels=channels)
# Delete channels
return surgeon.operate()
def prune(model, thresh=0.1):
'''
model: model to prune
thresh: neurons, corresponding to BSFilter layers lower
than thresh will be removed
'''
bs_idx = []
for idx, l in enumerate(model.layers):
if l.__class__.__name__ == "BSFilter":
bs_idx.append(idx)
weights = []
surgeon = Surgeon(model)
for i in bs_idx:
weights = model.layers[i].get_weights()[0]
l = model.layers[i]
to_prune = model.layers[i - 1]
surgeon.add_job('delete_layer', l)
surgeon.add_job('delete_channels',
to_prune,
channels=np.where(weights < thresh)[0].tolist())
return surgeon.operate()
def Prun_channel_1(self, layer, layer_1, config, channels_p):
weight = layer.weight
op_type = layer.type
op_name = layer.name
assert 0 <= config.get('sparsity') < 1
assert op_type in ['Conv1D', 'Conv2D']
assert op_type in config['op_types']
# op_name = layer.name
# assert 0 <= config.get('sparsity') < 1
# assert op_type in ['Conv1D', 'Conv2D']
# assert op_type in config['op_types']
#if layer.name in self.epoch_pruned_layers:
# assert layer.name in self.mask_dict
# return self.mask_dict.get(layer.name)
try:
w = tf.stop_gradient(tf.transpose(tf.reshape(weight, (-1, weight.shape[-1])), [1, 0]))
masks = np.ones(w.shape)
num_filters = w.shape[0]
num_prune = int(num_filters * config.get('sparsity'))
if num_filters < 2 or num_prune < 1:
return masks
#min_gm_idx = self._get_min_gm_kernel_idx_m(w, num_prune)
surgeon = Surgeon(self.bound_model, copy=False)
channels = channels_p
surgeon.add_job('delete_channels', layer_1, channels=channels)
#for idx in min_gm_idx:
# masks[idx] = 0.
finally:
masks = tf.reshape(tf.transpose(masks, [1, 0]), weight.shape)
masks = tf.Variable(masks)
self.mask_dict.update({op_name: masks})
self.epoch_pruned_layers.add(layer.name)
return surgeon.operate()
def random_pruning(self, P):
from kerassurgeon import Surgeon
surgeon = Surgeon(self.model)
layer_list = self.model.layers # list of layers in the model
to_prune_list = []
for i in range(len(layer_list)):
if 'conv' in str(layer_list[i]): #select convolutional layers only
to_prune_list.append(i) # gets their indexes
print(to_prune_list)
n_filters = []
for i in to_prune_list:
# gets the number of convolutional filters in every layer
n_filters.append(
self.model.get_layer(index=i).get_config()['filters'])
mega_list = [
] #mega list is a list of list where every element are the index of the filter removed in that layer
print(n_filters)
for kk in range(len(to_prune_list)): # for every conv layer
num_filter_layer = n_filters[kk] # number of filters in that layer
random_index = [] # index of the filters to be pruned
for jj in range(int(np.ceil(
P * num_filter_layer))): # gets random indexes
tmp = np.random.randint(0, num_filter_layer)
while tmp in random_index:
tmp = np.random.randint(
0, num_filter_layer
) # if the filter has alread benn choosen, try again
random_index.append(tmp)
mega_list.append(random_index)
layerr = self.model.layers[to_prune_list[kk]]
surgeon.add_job('delete_channels', layerr, channels=random_index)
self.model = surgeon.operate()
return mega_list
def prune_model_by_layer(model, percent_channels_delete, image_dir,
image_lists, preprocess_fct):
start = None
end = None
#model.summary()
# Create the Surgeon and add a 'delete_channels' job for each layer
# whose channels are to be deleted.
surgeon = Surgeon(model, copy=True)
for layer in model.layers[start:end]:
if identify.is_conv(layer):
print(layer.name)
num_total_channels = layer.output_shape[-1]
num_removed_channels = int(
num_total_channels * percent_channels_delete / 100) // 16 * 16
if num_removed_channels > 0:
if False:
images, y = get_images(image_dir, image_lists,
preprocess_fct)
#channels_importance=get_channels_apoz_importance(model, layer, images)
#channels_importance=get_channels_importance(model, layer, images,y)
#channels_importance=get_channels_loss(model, layer, images,y)
#channels_importance=get_channels_gradients(model, layer, images,y)
#channels_importance=get_channels_importance_with_gradient(model, layer, images,y)
else:
channels_importance = get_channels_l1_norm(model, layer)
total_channels_sorted = np.argsort(channels_importance)
#print('channels_importance:',channels_importance.shape,total_channels_sorted[:5])
channels = total_channels_sorted[:num_removed_channels]
print('before add_job:',
layer.name, channels, channels_importance[channels],
len(channels), num_total_channels)
surgeon.add_job('delete_channels', layer, channels=channels)
# Delete channels
return surgeon.operate()
def pruning(self, rank, vec, metodo, x_, y, classi=[]):
lunghezza = len(rank)
if type(
vec
) == list: # vec can contain either the percentage of filters to remove or their exact number
if type(vec[0]) == int:
type_flag = 0
elif type(vec[0]) == float:
type_flag = 1
else:
type_flag = 2
if lunghezza == 3: # cm_c criterion is used for the rank
mega_norm0 = rank[0]
mega_norm1 = rank[1]
mega_norm2 = rank[2]
n_filters = [
64, 64, 128, 128, 256, 256, 256, 512, 512, 512, 512, 512, 512
] # number of filters in vgg 16
un_arr0 = [oo for o in mega_norm0 for oo in o]
un_arr1 = [oo for o in mega_norm1 for oo in o]
un_arr2 = [oo for o in mega_norm2 for oo in o]
if metodo == 0: # rank is used layer by layer with no training in between
surgeon = Surgeon(self.model)
B = classi[0]
M = classi[1]
layer = [
lay for lay in self.model.layers if 'conv' in lay.name
]
for idx, i in enumerate(
layer
): #get the best and worst filters for every class.
best0 = list(
np.argsort(
mega_norm0[idx])[-int(B * len(mega_norm0[idx]))::])
best1 = list(
np.argsort(
mega_norm1[idx])[-int(B * len(mega_norm0[idx]))::])
best2 = list(
np.argsort(
mega_norm2[idx])[-int(B * len(mega_norm0[idx]))::])
worst0 = list(
np.argsort(
mega_norm0[idx])[0:int(M * len(mega_norm0[idx]))])
worst1 = list(
np.argsort(
mega_norm1[idx])[0:int(M * len(mega_norm0[idx]))])
worst2 = list(
np.argsort(
mega_norm2[idx])[0:int(M * len(mega_norm0[idx]))])
to_prune = []
for iii in worst0:
if (iii not in best1) and (iii not in best2):
if iii not in to_prune:
to_prune.append(iii)
for iii in worst1:
if (iii not in best0) and (iii not in best2):
if iii not in to_prune:
to_prune.append(iii)
for iii in worst2:
if (iii not in best0) and (iii not in best1):
if iii not in to_prune:
to_prune.append(iii)
print('number of filters to remove=', len(to_prune))
surgeon.add_job('delete_channels',
layer[idx],
channels=to_prune)
self.model = surgeon.operate()
elif metodo == 1: # rank is used globally with no training in between
surgeon = Surgeon(self.model)
B = classi[0]
M = classi[1]
best0 = list(
np.argsort(un_arr0)[-int(B *
sum(self.check_nfilters()))::])
best1 = list(
np.argsort(un_arr1)[-int(B *
sum(self.check_nfilters()))::])
best2 = list(
np.argsort(un_arr2)[-int(B *
sum(self.check_nfilters()))::])
worst0 = list(
np.argsort(un_arr0)[0:int(M * sum(self.check_nfilters()))])
worst1 = list(
np.argsort(un_arr1)[0:int(M * sum(self.check_nfilters()))])
worst2 = list(
np.argsort(un_arr2)[0:int(M * sum(self.check_nfilters()))])
lay = [i for i in self.model.layers if 'conv' in i.name]
for indice, hj in enumerate(lay):
to_prune = []
for iii in worst0:
if (iii not in best1) and (iii not in best2):
if un_arr0[iii] in mega_norm0[indice]:
tmp = np.argwhere(
mega_norm0[indice] == un_arr0[iii])
for gt in tmp:
if gt not in to_prune and len(
to_prune) < int(
n_filters[indice] * 0.95):
# used to prevent a whole network to be pruned
to_prune.append(int(gt))
for iii in worst1:
if (iii not in best0) and (iii not in best2):
if un_arr1[iii] in mega_norm1[indice]:
tmp = np.argwhere(
mega_norm1[indice] == un_arr1[iii])
for gt in tmp:
if gt not in to_prune and len(
to_prune) < int(
n_filters[indice] * 0.95):
to_prune.append(int(gt))
for iii in worst2:
if (iii not in best0) and (iii not in best1):
if un_arr2[iii] in mega_norm2[indice]:
tmp = np.argwhere(
mega_norm2[indice] == un_arr2[iii])
for gt in tmp:
if gt not in to_prune and len(
to_prune) < int(
n_filters[indice] * 0.95):
to_prune.append(int(gt))
surgeon.add_job('delete_channels', hj, channels=to_prune)
print('number of filters to remove=', len(to_prune))
self.model = surgeon.operate()
elif metodo == 2: # rank is used layer by layer but there is a training for every pruning step
surgeon = Surgeon(self.model)
B = classi[0]
M = classi[1]
save_path = os.path.join(os.getcwd(), 'saved_model_tmp.h5')
self.model.save(save_path) # save
for idx in range(12, -1, -1):
self.model = load_model(save_path)
trainable_lay = [0 for i in range(16)
] # all layers are trainable
trainable_lay[idx] = 1
self.set_trainable_layers(trainable_lay)
surgeon = Surgeon(self.model)
layer = [
jj for jj in self.model.layers if 'conv' in jj.name
]
best0 = list(
np.argsort(
mega_norm0[idx])[-int(B * len(mega_norm0[idx]))::])
best1 = list(
np.argsort(
mega_norm1[idx])[-int(B * len(mega_norm0[idx]))::])
best2 = list(
np.argsort(
mega_norm2[idx])[-int(B * len(mega_norm0[idx]))::])
worst0 = list(
np.argsort(
mega_norm0[idx])[0:int(M * len(mega_norm0[idx]))])
worst1 = list(
np.argsort(
mega_norm1[idx])[0:int(M * len(mega_norm0[idx]))])
worst2 = list(
np.argsort(
mega_norm2[idx])[0:int(M * len(mega_norm0[idx]))])
to_prune = []
for iii in worst0:
if (iii not in best1) and (iii not in best2):
if iii not in to_prune:
to_prune.append(iii)
for iii in worst1:
if (iii not in best0) and (iii not in best2):
if iii not in to_prune:
to_prune.append(iii)
for iii in worst2:
if (iii not in best0) and (iii not in best1):
if iii not in to_prune:
to_prune.append(iii)
print('number of filters to remove=', len(to_prune))
surgeon.add_job('delete_channels',
layer[idx],
channels=to_prune)
self.model = surgeon.operate()
sgd = optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True)
self.model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
for i in range(5): #
self.model.fit(x_, y, epochs=3, batch_size=8)
performance = self.model.evaluate(x_, y)
if performance[1] > 0.9:
break
self.model.save(save_path)
del self.model
K.clear_session()
tf.reset_default_graph()
self.model = load_model(save_path)
elif lunghezza != 3: # l1 and apz criteria
if metodo == 0:
surgeon = Surgeon(self.model)
layer = [jj for jj in self.model.layers if 'conv' in jj.name]
un_arr0 = [oo for o in rank for oo in o]
for idx in range(13):
norm = rank[idx]
if type_flag == 0:
indici = np.array(norm).argsort()[0:vec[idx]]
elif type_flag == 1:
indici = np.array(
norm).argsort()[0:int(vec[idx] * len(rank[idx]))]
to_remove_channel = indici.tolist()
surgeon.add_job('delete_channels',
layer[idx],
channels=indici)
self.model = surgeon.operate()
elif metodo == 1:
surgeon = Surgeon(self.model)
un_arr = [oo for o in rank for oo in o]
layer = [jj for jj in self.model.layers if 'conv' in jj.name]
if type_flag == 0:
norm = np.argsort(un_arr)[0:sum(vec)] # ordered vector
elif type_flag == 1: # percentage is used
norm = np.argsort(
un_arr)[0:int(np.sum(self.check_nfilters()) * vec[0])]
print('len norm=', len(norm))
for idx, jj in enumerate(layer):
to_prune = []
for kk in norm:
if un_arr[kk] in rank[idx]:
tmp = np.argwhere(
rank[idx] == un_arr[kk]).squeeze()
try:
if int(tmp) not in to_prune and len(
to_prune) < int(n_filters[idx] * 0.95):
to_prune.append(int(tmp))
except:
for ij in tmp:
if ij not in to_prune and len(
to_prune) < int(
n_filters[idx] * 0.95):
to_prune.append(ij)
print('len(to_prune)', len(to_prune), idx)
surgeon.add_job('delete_channels',
layer[idx],
channels=to_prune)
self.model = surgeon.operate()
elif metodo == 2: # faccio un train per ogni strato
surgeon = Surgeon(self.model)
self.model.save('R:\\saved_model_tmp.h5')
for idx in range(12, -1, -1):
self.model = load_model('R:\\saved_model_tmp.h5')
# trainable_lay=[0 for i in range(16)]
# trainable_lay[idx]=1
# self.set_trainable_layers(trainable_lay)
surgeon = Surgeon(self.model)
layer = [
jj for jj in self.model.layers if 'conv' in jj.name
]
norm = rank[idx]
if type_flag == 0: #lavoro direttamente con il numero di filtri
indici = np.array(norm).argsort()[0:vec[idx]]
print('len(indici)=', len(indici))
elif type_flag == 1: # specifico solo la percentuale
indici = np.array(
norm).argsort()[0:int(vec[idx] * len(rank[idx]))]
print('len(indici)=', len(indici))
elif type_flag == 2:
#specifico la distanza da una sigma
std = np.std(norm)
index = (norm - np.man(norm)) < -std
indici = np.argwhere(index == 1)
print('number of filters to remove=', len(indici))
surgeon.add_job('delete_channels',
layer[idx],
channels=indici)
self.model = surgeon.operate()
sgd = optimizers.SGD(lr=0.001, momentum=0.9, nesterov=True)
self.model.compile(loss='categorical_crossentropy',
optimizer=sgd,
metrics=['accuracy'])
for i in range(5):
self.model.fit(x_, y, epochs=3, batch_size=8)
performance = self.model.evaluate(x_, y)
if performance[1] > 0.9:
break
save_path = os.path.join(os.getcwd(), 'saved_model_tmp.h5')
self.model.save(save_path)
del self.model
K.clear_session()
tf.reset_default_graph()
self.model = load_model(save_path)
def pruning_method_conv(model, layer_to_prune, pruning_amount, method):
if method == 'L1norm':
# Load surgeon package
surgeon = Surgeon(model)
# Store weights from conv layers [0][1] = [weight][bias] Hence, [0] to store weights only
conv_layer_weights = [
model.layers[i].get_weights()[0] for i in layer_to_prune
]
print('number of layer to prune: ' + str(len(conv_layer_weights)))
for i in range(len(conv_layer_weights)):
if pruning_amount[i] == 0:
continue
weight = conv_layer_weights[i]
num_filters = len(weight[0, 0, 0, :])
print('total number of filter: ' + str(num_filters))
weight_removable = {}
# compute L1-nom of each filter weight and store it in a dictionary(weight_removable)
for j in range(num_filters):
L1_norm = np.sum(abs(weight[:, :, :, j]))
filter_number = 'filter_{}'.format(j)
weight_removable[filter_number] = L1_norm
# sort the filter according to the ascending L1 value
weight_removable_sort = sorted(weight_removable.items(),
key=lambda kv: kv[1])
# 'filter_24'(string) -> 24(int),
# extracting filter number from '(filter_2, 0.515..), eg) extracting '2' from '(filter_2, 0.515..)
remove_channel = [
int(weight_removable_sort[i][0].split("_")[1])
for i in range(0, pruning_amount[i])
]
# delete filters with lowest scores
surgeon.add_job('delete_channels',
model.layers[layer_to_prune[i]],
channels=remove_channel)
model_pruned = surgeon.operate()
return model_pruned
elif method == 'geometric_median_conv':
# Load surgeon package
surgeon = Surgeon(model)
# Store weights from conv layers [0][1] = [weight][bias] Hence, [0] to store weights only
conv_layer_weights = [
model.layers[i].get_weights()[0] for i in layer_to_prune
]
print('number of layer to prune: ' + str(len(conv_layer_weights)))
for i in range(len(conv_layer_weights)):
if pruning_amount[i] == 0:
continue
weight = conv_layer_weights[i]
num_filters = len(weight[0, 0, 0, :])
print('total number of filter: ' + str(num_filters))
weight_removable = {}
# 1. Reduce dimension 4D -> 2D
# 2. Normalization (L1, L2)
# 3. Sort norm value => get index order
# 4. Sort weight by index order
# 5. Calculate distance between coordinates
# 6. Sum distance (of each filter)
norm_val = geometric_median(weight, "euclidean", "L1")
# compute L1-nom of each filter weight and store it in a dictionary(weight_removable)
for j in range(num_filters):
filter_number = 'filter_{}'.format(j)
weight_removable[filter_number] = norm_val[j]
# sort the filter according to the ascending L1 value
weight_removable_sort = sorted(weight_removable.items(),
key=lambda kv: kv[1])
# print(weight_removable_sort)
# 'filter_24'(string) -> 24(int),
# extracting filter number from '(filter_2, 0.515..), eg) extracting '2' from '(filter_2, 0.515..)
remove_channel = [
int(weight_removable_sort[i][0].split("_")[1])
for i in range(0, pruning_amount[i])
]
# print(remove_channel)
# delete filters with lowest scores
surgeon.add_job('delete_channels',
model.layers[layer_to_prune[i]],
channels=remove_channel)
model_pruned = surgeon.operate()
return model_pruned
