Exemple #1
0
def random_conv_global(model, psize=0.9):
    """ channel wise global pruning """
    layer_start = {}
    pidx_dict = {}
    sidx_dict = {}
    node_num = 0
    for l in range(len(model.layers)):
        layer = model.layers[l]
        layer_class = layer.__class__.__name__
        if not (layer_class == 'Conv2D' or layer_class == 'Dense'):
            continue
        elif layer.name == 'conv0':
            continue
        elif layer.name == 'dense1':
            continue
        layer_start[layer.name] = node_num
        pidx_dict[layer.name] = []
        n_node = layer.outut_shape[-1]
        for i in range(n_node):
            sidx_dict[i+node_num] = layer.name
        node_num += n_node
    idx_rand = np.arange(node_num)
    idx_rand = np.random.shuffle(idx_rand)
    psize = int(round(psize*node_num))
    pidx = idx_rand[:psize]
    for p in pidx:
        layer_name = sidx_dict[p]
        pidx_dict[layer_name].append(p-layer_start[layer_name])
    for name, p_idx in pidx_dict.items():
        if len(p_idx) == 0:
            continue
        print("Pruning layer "+name+" channels: ", p_idx)
        model = delete_channels(model, model.get_layer(name=name), p_idx)
    return model
Exemple #2
0
def grad_activation_rank(model, input_img, name="conv3",psize=0.5):
    """ use gradient as sign of activation """
    layer = model.get_layer(name=name)
    n_node = layer.output_shape[-1]
    psize = int(round(psize*n_node))
    inputs = model.input
    out_shape = layer.output_shape
    if len(out_shape)>2:
        mean_axis = (0,1,2)
    else:
        mean_axis = 0
    mean = K.mean(layer.output, axis=mean_axis)
    print("mean shape:", mean.shape)
    std_grads = []
    for i in range(n_node):
        print("grads[{}]".format(i))
        grads = K.gradients(mean[i], inputs)[0]
        grads = K.sum(K.abs(grads))
        iterate = K.function([inputs],[grads])
        grads = iterate([input_img])
        std_grads.append(grads[0])
    print("std_grads:", std_grads)
    pidx = np.argsort(std_grads)
    print(pidx)
    pidx = pidx[:psize]
    model = delete_channels(model, layer, pidx.tolist())
    return model
def prune_one_layer(model, pruned_indexes, layer_ix, opt):
    """Prunes one layer based on a Keras Model, layer index and indexes of filters to prune"""
    model_pruned = delete_channels(model, model.layers[layer_ix],
                                   pruned_indexes)
    model_pruned.compile(loss='categorical_crossentropy',
                         optimizer=opt,
                         metrics=['accuracy'])
    return model_pruned
Exemple #4
0
def random_conv_channel(model, name="conv3", psize=0.5):
    """ channel wise random pruning 
        for conv layer
    """
    n_filter = model.get_layer(name=name).output_shape[-1]
    n_node = model.get_layer(name=name).output_shape[-1]
    psize = int(round(psize*n_node))
    pidx = np.arange(n_filter)
    np.random.shuffle(pidx)
    pidx = pidx[:psize]
    model = delete_channels(model, model.get_layer(name=name), pidx.tolist())
    return model
Exemple #5
0
def zero_weight(model, layer_name, psize):
    print("Estimate weight in DNN, layer[{}]".format(layer_name))
    layer = model.get_layer(name=layer_name)
    print("layer type = ", layer.__class__.__name__)
    weights = layer.get_weights()
    weight = weights[0]
    print("weight shape ", weight.shape)
    node_sum = np.sum(np.absolute(weight), axis=0)
    idx_sort = np.argsort(node_sum, axis=-1)
    pidx = idx_sort[:psize]
    print("pruning channel:", pidx)
    model = delete_channels(model, layer, pidx.tolist())
    return model
def _main():
    log_dir = './logs/006MorePercent_Prune_net4_rain/001/'
    classes_path = 'model_data/coco_classes.txt'
    anchors_path = 'model_data/tiny_yolo_anchors.txt'

    class_names = get_classes(classes_path)
    num_classes = len(class_names)
    anchors = get_anchors(anchors_path)

    input_shape = (416, 416)
    conv_lyr_idx = ["conv2d_1", "conv2d_2", "conv2d_3", "conv2d_4", "conv2d_5", "conv2d_6",
                    "conv2d_7", "conv2d_8", "conv2d_11", "conv2d_9", "conv2d_12", "conv2d_10",
                    "conv2d_13"]
    conv_lyr_output_dim = [16,   32,        64,          128,        256,        512,
                           1024, 256,       128,         512,        256]

    # # 002 5.3Mb prune rates
    prune_percentages =   [0,    0,          0,          0.5,        0.5,        0.5,
                           0.75, 0.5,        0.75,       0.75,       0.75]

    sort_idx_L = np.load(log_dir+"grad_am_sort_idx_L.npy", allow_pickle=True)

    # prune the original yolo-v3-tiny
    model = create_tiny_model(input_shape, anchors, num_classes, log_dir + 'yolov3-tiny.h5')

    # total_channels = get_total_channels(model)
    conv_layer_names = []
    for layer in model.layers:
        if layer.__class__.__name__ == 'Conv2D':
            # if layer.output_shape[-1] > 32:
            conv_layer_names.append(layer.name)

    for i_selected_lyr in range(len(sort_idx_L) - 2):
        conv_layer_name = conv_layer_names[i_selected_lyr]
        conv_layers = model.get_layer(name=conv_layer_name)
        sort_idx = sort_idx_L[i_selected_lyr]
        prune_percentage = prune_percentages[i_selected_lyr]

        prune_num = int(len(sort_idx) * prune_percentage)
        prune_channels = sort_idx[:prune_num]
        prune_channels = prune_channels.tolist()
        model = delete_channels(model, conv_layers, prune_channels)

    save_pruned_net_name = log_dir + 'GradAM_pruned_yolo.h5'
    # Clean up tensorflow session after pruning and re-load model
    model.save_weights(save_pruned_net_name)

    del model
    K.clear_session()
    tf.reset_default_graph()
Exemple #7
0
def zero_channels_all(model, psize=0.5):
    """ ranking across all conv layers """
    node_sums = []
    layer_start = {}
    pidx_dict = {}
    sidx_dict = {}
    node_num = 0
    for l in range(len(model.layers)):
        layer = model.layers[l]
        layer_class = layer.__class__.__name__
        if not layer_class == 'Conv2D':
            continue
        elif layer.name == 'conv0':
            continue
        layer_start[layer.name] = node_num
        n_node = layer.output_shape[-1]
        pidx_dict[layer.name] = []
        for i in range(n_node):
            sidx_dict[i+node_num] = layer.name
        node_num += n_node
        weights = layer.get_weights()
        weight = np.array(weights[0])
        ch_num = weight.shape[3]
        w_sum = np.zeros(ch_num)
        for i in range(ch_num):
            w_sum[i] = np.sum(np.absolute(weight[:,:,:,i]))
        w_sum /= np.sqrt(np.mean(np.square(w_sum), keepdims=True))
        if len(node_sums)==0:
            node_sums = w_sum.tolist()
        else:
            node_sums += w_sum.tolist()
    idx_sort = np.argsort(node_sums, axis=-1)
    print("idx_sort:" ,idx_sort)
    psize = int(round(psize*node_num))
    pidx = idx_sort[:psize]
    for p in pidx:
        name = sidx_dict[p]
        pidx_dict[name].append(p-layer_start[name])
    for name, p_idx in pidx_dict.items():
        print("pruning "+name+" prune channels:", p_idx)
        if len(p_idx) >= model.get_layer(name=name).output_shape[-1]:
            p_idx.pop()
        model = delete_channels(model, model.get_layer(name=name), p_idx)
    return model
Exemple #8
0
def zero_channels(model, layer_name, psize):
    print("Estimate weight in CNN, layer[{}]".format(layer_name))
    layer = model.get_layer(name=layer_name)
    weights = layer.get_weights()
    print("CNN weight dim axis=0 : ", len(weights))
    # weights[0] shape = (ch_dim,ch_dim, input_ch_num,ch_num)
    # weights[1] shape = (ch_num,)
    weight = np.array(weights[0])
    print("CNN weights[0] shape:", weight.shape)
    ch_num = weight.shape[3]
    w_sum = np.zeros(ch_num)
    for i in range(ch_num):
        w_sum[i] = np.sum(np.absolute(weight[:, :, :, i]))
    print("sum shape:", w_sum.shape)
    idx_sort = np.argsort(w_sum, axis=-1)
    pidx = idx_sort[:psize]
    print("CNN pruning channel:", pidx)
    model = delete_channels(model, layer, pidx.tolist())
    return model
Exemple #9
0
def zero_weight_all(model, psize=0.5):
    """ ranking across all dense layers """
    node_sums = []
    layer_start = {}
    pidx_dict = {}
    sidx_dict = {}
    node_num = 0
    for l in range(len(model.layers)):
        layer = model.layers[l]
        layer_class = layer.__class__.__name__
        if not layer_class == 'Dense':
            continue
        elif layer.name == 'dense_o':
            continue
        layer_start[layer.name] = node_num
        pidx_dict[layer.name] = []
        n_node = layer.output_shape[-1]
        for i in range(n_node):
            sidx_dict[i+node_num] = layer.name
        node_num += n_node
        weights = layer.get_weights()
        weight = weights[0]
        node_weight = np.sum(np.absolute(weight), axis=0)
        node_weight /= np.sqrt(np.mean(np.square(node_weight), keepdims=True))
        if len(node_sums)==0:
            node_sums = node_weight.tolist()
        else:
            node_sums += node_weight.tolist()
    idx_sort = np.argsort(node_sums, axis=-1)
    psize = int(round(psize*node_num))
    pidx = idx_sort[:psize]
    for p in pidx:
        layer_name = sidx_dict[p]
        pidx_dict[layer_name].append(p-layer_start[layer_name])
    for name, p_idx in pidx_dict.items():
        if len(p_idx) == 0:
            continue
        elif len(p_idx) >= model.get_layer(name=name).output_shape[-1]:
            p_idx.pop()
        print("prune layer "+name+" for nodes: ", p_idx)
        model = delete_channels(model, model.get_layer(name=name), p_idx)
    return model
Exemple #10
0
def mean_activation_rank(model, input_img, name="conv3", psize=0.5):
    """
    use mean of layer output as sign of activation
    for pruning conv layer
    psize: floating point 0 to 1
    """
    layer = model.get_layer(name=name)
    n_node = layer.output_shape[-1]
    psize = int(round(psize*n_node))
    out_shape = layer.output_shape
    if len(out_shape)>2:
        mean_axis = (0,1,2)
    else:
        mean_axis = 0
    inputs = model.input
    mean = K.mean(model.get_layer(name=name).output, axis=mean_axis)
    iterate = K.function([inputs], [mean])
    mean_values = iterate([input_img])
    print("mean values:", mean_values)
    idx_sort = np.argsort(mean_values)[0]
    pidx = idx_sort[:psize]
    print(pidx)
    model = delete_channels(model, model.get_layer(name=name),pidx.tolist())
    return model
Exemple #11
0
def grad_layerwise_rank(model, input_img, psize=1):
    """ 
    use gradient for layer-wise ranking 
    psize is an int indicating how many layers to prune
    """
    layer_idx = []
    layer_grads = []
    lidx = 0
    for layer in model.layers:
        layer_class = layer.__class__.__name__
        if layer_class == 'Conv2D':
            layer_idx.append(lidx)
            lidx += 1
        elif layer_class == 'Dense':
            layer_idx.append(lidx)
            lidx += 1
        else:
            lidx += 1
            continue
        inputs = model.input
        mean = K.mean(layer.output)
        #print("mean :", mean)
        grads = K.gradients(mean, inputs)[0]
        grads = K.sum(K.abs(grads))
        print("grads{} shape:{}".format(layer_class, grads.shape))
        iterate = K.function([inputs, K.learning_phase()],[grads])
        grads = iterate([input_img, 0])
        layer_grads.append(grads[0])
    print("layer grads:",layer_grads)
    pidx = np.argsort(layer_grads)
    pidx = np.delete(pidx, [i for i in pidx if pidx[i] == 0])
    #new_pidx = []
    #for p in pidx:
    #    new_pidx.append(layer_idx[p])
    #print("prune idx:", new_pidx)
    pidx = pidx[:psize]
    pidx = pidx.tolist()
    print("pruning idx:", pidx)
    # pruning according to input and output size
    for i in pidx:
        layer = model.layers[layer_idx[i]]
        layer_class = layer.__class__.__name__
        if i == 0:
            input_size = layer.input_shape
        else:
            input_size = model.layers[layer_idx[i-1]].output_shape
        if i == len(layer_idx)-1:
            output_size = layer.output_shape
        else:
            output_size = model.layers[layer_idx[i+1]].input_shape
        if input_size[-1] > output_size[-1]:
            if i == 0:
                # we do not delete input layer in dnn
                continue
            # former layer lager than latter layer
            # we prune former layer to make them fit
            psize_chnl = input_size[-1] - output_size[-1]
            last_layer = model.layers[layer_idx[i-1]]
            model = delete_channels(model, last_layer, np.arange(psize_chnl).tolist())
            #model = delete_layer(model, layer)
        elif input_size[-1] < output_size[-1]:
            # former layer smaller than latter
            # happens in CNN
            # we prune this layer to make them fit
            psize_chnl = output_size[-1] - input_size[-1]
            model = delete_channels(model, layer, np.arange(psize_chnl).tolist())
        print("Pruning layer {}".format(layer.name))
        model = delete_layer(model, layer)
        return model