def random_conv_global(model, psize=0.9):
""" channel wise global pruning """
layer_start = {}
pidx_dict = {}
sidx_dict = {}
node_num = 0
for l in range(len(model.layers)):
layer = model.layers[l]
layer_class = layer.__class__.__name__
if not (layer_class == 'Conv2D' or layer_class == 'Dense'):
continue
elif layer.name == 'conv0':
continue
elif layer.name == 'dense1':
continue
layer_start[layer.name] = node_num
pidx_dict[layer.name] = []
n_node = layer.outut_shape[-1]
for i in range(n_node):
sidx_dict[i+node_num] = layer.name
node_num += n_node
idx_rand = np.arange(node_num)
idx_rand = np.random.shuffle(idx_rand)
psize = int(round(psize*node_num))
pidx = idx_rand[:psize]
for p in pidx:
layer_name = sidx_dict[p]
pidx_dict[layer_name].append(p-layer_start[layer_name])
for name, p_idx in pidx_dict.items():
if len(p_idx) == 0:
continue
print("Pruning layer "+name+" channels: ", p_idx)
model = delete_channels(model, model.get_layer(name=name), p_idx)
return model
def grad_activation_rank(model, input_img, name="conv3",psize=0.5):
""" use gradient as sign of activation """
layer = model.get_layer(name=name)
n_node = layer.output_shape[-1]
psize = int(round(psize*n_node))
inputs = model.input
out_shape = layer.output_shape
if len(out_shape)>2:
mean_axis = (0,1,2)
else:
mean_axis = 0
mean = K.mean(layer.output, axis=mean_axis)
print("mean shape:", mean.shape)
std_grads = []
for i in range(n_node):
print("grads[{}]".format(i))
grads = K.gradients(mean[i], inputs)[0]
grads = K.sum(K.abs(grads))
iterate = K.function([inputs],[grads])
grads = iterate([input_img])
std_grads.append(grads[0])
print("std_grads:", std_grads)
pidx = np.argsort(std_grads)
print(pidx)
pidx = pidx[:psize]
model = delete_channels(model, layer, pidx.tolist())
return model
def prune_one_layer(model, pruned_indexes, layer_ix, opt):
"""Prunes one layer based on a Keras Model, layer index and indexes of filters to prune"""
model_pruned = delete_channels(model, model.layers[layer_ix],
pruned_indexes)
model_pruned.compile(loss='categorical_crossentropy',
optimizer=opt,
metrics=['accuracy'])
return model_pruned
def random_conv_channel(model, name="conv3", psize=0.5):
""" channel wise random pruning
for conv layer
"""
n_filter = model.get_layer(name=name).output_shape[-1]
n_node = model.get_layer(name=name).output_shape[-1]
psize = int(round(psize*n_node))
pidx = np.arange(n_filter)
np.random.shuffle(pidx)
pidx = pidx[:psize]
model = delete_channels(model, model.get_layer(name=name), pidx.tolist())
return model
def zero_weight(model, layer_name, psize):
print("Estimate weight in DNN, layer[{}]".format(layer_name))
layer = model.get_layer(name=layer_name)
print("layer type = ", layer.__class__.__name__)
weights = layer.get_weights()
weight = weights[0]
print("weight shape ", weight.shape)
node_sum = np.sum(np.absolute(weight), axis=0)
idx_sort = np.argsort(node_sum, axis=-1)
pidx = idx_sort[:psize]
print("pruning channel:", pidx)
model = delete_channels(model, layer, pidx.tolist())
return model
def _main():
log_dir = './logs/006MorePercent_Prune_net4_rain/001/'
classes_path = 'model_data/coco_classes.txt'
anchors_path = 'model_data/tiny_yolo_anchors.txt'
class_names = get_classes(classes_path)
num_classes = len(class_names)
anchors = get_anchors(anchors_path)
input_shape = (416, 416)
conv_lyr_idx = ["conv2d_1", "conv2d_2", "conv2d_3", "conv2d_4", "conv2d_5", "conv2d_6",
"conv2d_7", "conv2d_8", "conv2d_11", "conv2d_9", "conv2d_12", "conv2d_10",
"conv2d_13"]
conv_lyr_output_dim = [16, 32, 64, 128, 256, 512,
1024, 256, 128, 512, 256]
# # 002 5.3Mb prune rates
prune_percentages = [0, 0, 0, 0.5, 0.5, 0.5,
0.75, 0.5, 0.75, 0.75, 0.75]
sort_idx_L = np.load(log_dir+"grad_am_sort_idx_L.npy", allow_pickle=True)
# prune the original yolo-v3-tiny
model = create_tiny_model(input_shape, anchors, num_classes, log_dir + 'yolov3-tiny.h5')
# total_channels = get_total_channels(model)
conv_layer_names = []
for layer in model.layers:
if layer.__class__.__name__ == 'Conv2D':
# if layer.output_shape[-1] > 32:
conv_layer_names.append(layer.name)
for i_selected_lyr in range(len(sort_idx_L) - 2):
conv_layer_name = conv_layer_names[i_selected_lyr]
conv_layers = model.get_layer(name=conv_layer_name)
sort_idx = sort_idx_L[i_selected_lyr]
prune_percentage = prune_percentages[i_selected_lyr]
prune_num = int(len(sort_idx) * prune_percentage)
prune_channels = sort_idx[:prune_num]
prune_channels = prune_channels.tolist()
model = delete_channels(model, conv_layers, prune_channels)
save_pruned_net_name = log_dir + 'GradAM_pruned_yolo.h5'
# Clean up tensorflow session after pruning and re-load model
model.save_weights(save_pruned_net_name)
del model
K.clear_session()
tf.reset_default_graph()
def zero_channels_all(model, psize=0.5):
""" ranking across all conv layers """
node_sums = []
layer_start = {}
pidx_dict = {}
sidx_dict = {}
node_num = 0
for l in range(len(model.layers)):
layer = model.layers[l]
layer_class = layer.__class__.__name__
if not layer_class == 'Conv2D':
continue
elif layer.name == 'conv0':
continue
layer_start[layer.name] = node_num
n_node = layer.output_shape[-1]
pidx_dict[layer.name] = []
for i in range(n_node):
sidx_dict[i+node_num] = layer.name
node_num += n_node
weights = layer.get_weights()
weight = np.array(weights[0])
ch_num = weight.shape[3]
w_sum = np.zeros(ch_num)
for i in range(ch_num):
w_sum[i] = np.sum(np.absolute(weight[:,:,:,i]))
w_sum /= np.sqrt(np.mean(np.square(w_sum), keepdims=True))
if len(node_sums)==0:
node_sums = w_sum.tolist()
else:
node_sums += w_sum.tolist()
idx_sort = np.argsort(node_sums, axis=-1)
print("idx_sort:" ,idx_sort)
psize = int(round(psize*node_num))
pidx = idx_sort[:psize]
for p in pidx:
name = sidx_dict[p]
pidx_dict[name].append(p-layer_start[name])
for name, p_idx in pidx_dict.items():
print("pruning "+name+" prune channels:", p_idx)
if len(p_idx) >= model.get_layer(name=name).output_shape[-1]:
p_idx.pop()
model = delete_channels(model, model.get_layer(name=name), p_idx)
return model
def zero_channels(model, layer_name, psize):
print("Estimate weight in CNN, layer[{}]".format(layer_name))
layer = model.get_layer(name=layer_name)
weights = layer.get_weights()
print("CNN weight dim axis=0 : ", len(weights))
# weights[0] shape = (ch_dim,ch_dim, input_ch_num,ch_num)
# weights[1] shape = (ch_num,)
weight = np.array(weights[0])
print("CNN weights[0] shape:", weight.shape)
ch_num = weight.shape[3]
w_sum = np.zeros(ch_num)
for i in range(ch_num):
w_sum[i] = np.sum(np.absolute(weight[:, :, :, i]))
print("sum shape:", w_sum.shape)
idx_sort = np.argsort(w_sum, axis=-1)
pidx = idx_sort[:psize]
print("CNN pruning channel:", pidx)
model = delete_channels(model, layer, pidx.tolist())
return model
def zero_weight_all(model, psize=0.5):
""" ranking across all dense layers """
node_sums = []
layer_start = {}
pidx_dict = {}
sidx_dict = {}
node_num = 0
for l in range(len(model.layers)):
layer = model.layers[l]
layer_class = layer.__class__.__name__
if not layer_class == 'Dense':
continue
elif layer.name == 'dense_o':
continue
layer_start[layer.name] = node_num
pidx_dict[layer.name] = []
n_node = layer.output_shape[-1]
for i in range(n_node):
sidx_dict[i+node_num] = layer.name
node_num += n_node
weights = layer.get_weights()
weight = weights[0]
node_weight = np.sum(np.absolute(weight), axis=0)
node_weight /= np.sqrt(np.mean(np.square(node_weight), keepdims=True))
if len(node_sums)==0:
node_sums = node_weight.tolist()
else:
node_sums += node_weight.tolist()
idx_sort = np.argsort(node_sums, axis=-1)
psize = int(round(psize*node_num))
pidx = idx_sort[:psize]
for p in pidx:
layer_name = sidx_dict[p]
pidx_dict[layer_name].append(p-layer_start[layer_name])
for name, p_idx in pidx_dict.items():
if len(p_idx) == 0:
continue
elif len(p_idx) >= model.get_layer(name=name).output_shape[-1]:
p_idx.pop()
print("prune layer "+name+" for nodes: ", p_idx)
model = delete_channels(model, model.get_layer(name=name), p_idx)
return model
def mean_activation_rank(model, input_img, name="conv3", psize=0.5):
"""
use mean of layer output as sign of activation
for pruning conv layer
psize: floating point 0 to 1
"""
layer = model.get_layer(name=name)
n_node = layer.output_shape[-1]
psize = int(round(psize*n_node))
out_shape = layer.output_shape
if len(out_shape)>2:
mean_axis = (0,1,2)
else:
mean_axis = 0
inputs = model.input
mean = K.mean(model.get_layer(name=name).output, axis=mean_axis)
iterate = K.function([inputs], [mean])
mean_values = iterate([input_img])
print("mean values:", mean_values)
idx_sort = np.argsort(mean_values)[0]
pidx = idx_sort[:psize]
print(pidx)
model = delete_channels(model, model.get_layer(name=name),pidx.tolist())
return model
def grad_layerwise_rank(model, input_img, psize=1):
"""
use gradient for layer-wise ranking
psize is an int indicating how many layers to prune
"""
layer_idx = []
layer_grads = []
lidx = 0
for layer in model.layers:
layer_class = layer.__class__.__name__
if layer_class == 'Conv2D':
layer_idx.append(lidx)
lidx += 1
elif layer_class == 'Dense':
layer_idx.append(lidx)
lidx += 1
else:
lidx += 1
continue
inputs = model.input
mean = K.mean(layer.output)
#print("mean :", mean)
grads = K.gradients(mean, inputs)[0]
grads = K.sum(K.abs(grads))
print("grads{} shape:{}".format(layer_class, grads.shape))
iterate = K.function([inputs, K.learning_phase()],[grads])
grads = iterate([input_img, 0])
layer_grads.append(grads[0])
print("layer grads:",layer_grads)
pidx = np.argsort(layer_grads)
pidx = np.delete(pidx, [i for i in pidx if pidx[i] == 0])
#new_pidx = []
#for p in pidx:
# new_pidx.append(layer_idx[p])
#print("prune idx:", new_pidx)
pidx = pidx[:psize]
pidx = pidx.tolist()
print("pruning idx:", pidx)
# pruning according to input and output size
for i in pidx:
layer = model.layers[layer_idx[i]]
layer_class = layer.__class__.__name__
if i == 0:
input_size = layer.input_shape
else:
input_size = model.layers[layer_idx[i-1]].output_shape
if i == len(layer_idx)-1:
output_size = layer.output_shape
else:
output_size = model.layers[layer_idx[i+1]].input_shape
if input_size[-1] > output_size[-1]:
if i == 0:
# we do not delete input layer in dnn
continue
# former layer lager than latter layer
# we prune former layer to make them fit
psize_chnl = input_size[-1] - output_size[-1]
last_layer = model.layers[layer_idx[i-1]]
model = delete_channels(model, last_layer, np.arange(psize_chnl).tolist())
#model = delete_layer(model, layer)
elif input_size[-1] < output_size[-1]:
# former layer smaller than latter
# happens in CNN
# we prune this layer to make them fit
psize_chnl = output_size[-1] - input_size[-1]
model = delete_channels(model, layer, np.arange(psize_chnl).tolist())
print("Pruning layer {}".format(layer.name))
model = delete_layer(model, layer)
return model
