def plot_dead_filter_num_with_different_fdt(): # print() device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # # # net=create_net.vgg_cifar10() checkpoint = torch.load( '../data/baseline/resnet56_cifar10,accuracy=0.94230.tar') net = resnet_cifar.resnet56().to(device) net.load_state_dict(checkpoint['state_dict']) val_loader = data_loader.create_validation_loader(batch_size=500, num_workers=6, dataset_name='cifar10') relu_list, neural_list = evaluate.check_ReLU_alive(net=net, neural_dead_times=8000, data_loader=val_loader) # net=vgg.vgg16_bn(pretrained=False) # checkpoint=torch.load('/home/victorfang/Desktop/vgg16_bn_imagenet_deadReLU.tar') # net=resnet.resnet34(pretrained=True) # checkpoint=torch.load('/home/victorfang/Desktop/resnet34_imagenet_DeadReLU.tar') # neural_list=checkpoint['neural_list'] # relu_list=checkpoint['relu_list'] neural_dead_times = 8000 # neural_dead_times=40000 fdt_list = [0.001 * i for i in range(1, 1001)] dead_filter_num = [] for fdt in fdt_list: dead_filter_num.append( dead_filter_statistics(net=net, neural_list=neural_list, neural_dead_times=neural_dead_times, filter_FIRE=fdt, relu_list=relu_list)) if fdt == 0.8: print() plt.figure() plt.title('df') plt.plot(fdt_list, dead_filter_num) plt.xlabel('filter activation ratio') plt.ylabel('number of filters') plt.legend() plt.show()
def dead_neural_rate(): # checkpoint=torch.load('/home/victorfang/Desktop/vgg16_bn_imagenet_deadReLU.tar') # checkpoint=torch.load('/home/victorfang/Desktop/resnet34_imagenet_DeadReLU.tar') # neural_list=checkpoint['neural_list'] # relu_list=checkpoint['relu_list'] device = torch.device("cuda" if torch.cuda.is_available() else "cpu") checkpoint = torch.load( '../data/baseline/resnet56_cifar10,accuracy=0.94230.tar') net = resnet_cifar.resnet56().to(device) net.load_state_dict(checkpoint['state_dict']) # net=create_net.vgg_cifar10() val_loader = data_loader.create_validation_loader(batch_size=1000, num_workers=6, dataset_name='cifar10') # train_loader=data_loader.create_train_loader(batch_size=1600,num_workers=6,dataset_name='cifar10') # relu_list, neural_list = evaluate.check_ReLU_alive(net=net, neural_dead_times=10000, data_loader=val_loader) # ndt_list=[i for i in range(35000,51000,1000)] ndt_list = [i for i in range(6000, 11000, 1000)] dead_rate = [] for ndt in ndt_list: print(ndt) dead_rate.append( evaluate.cal_dead_neural_rate(neural_dead_times=ndt, neural_list_temp=neural_list)) plt.figure() plt.title('df') plt.plot(ndt_list, dead_rate) plt.xlabel('neural dead times') plt.ylabel('neuron dead rate%') plt.legend() plt.show()
def plot_dead_neuron_filter_number( neural_dead_times=8000, dataset_name='cifar10', ): fontsize = 17 label_fontsize = 24 tick_fontsize = 20 device = torch.device("cuda" if torch.cuda.is_available() else "cpu") checkpoint = torch.load( '../data/baseline/vgg16_bn_cifar10,accuracy=0.941.tar') vgg16 = storage.restore_net(checkpoint) vgg16.load_state_dict(checkpoint['state_dict']) checkpoint = torch.load( '../data/baseline/resnet56_cifar10,accuracy=0.94230.tar') resnet56 = resnet_cifar.resnet56().to(device) resnet56.load_state_dict(checkpoint['state_dict']) vgg16_imagenet = vgg.vgg16_bn(pretrained=True).to(device) checkpoint = torch.load( '/home/victorfang/Desktop/vgg16_bn_imagenet_deadReLU.tar') relu_list_imagenet = checkpoint['relu_list'] neural_list_imagenet = checkpoint['neural_list'] loader = data_loader.create_validation_loader(batch_size=100, num_workers=1, dataset_name=dataset_name) # loader=data_loader.create_validation_loader(batch_size=1000,num_workers=8,dataset_name='cifar10_trainset') relu_list_vgg, neural_list_vgg = evaluate.check_ReLU_alive( net=vgg16, neural_dead_times=neural_dead_times, data_loader=loader, max_data_to_test=10000) relu_list_resnet, neural_list_resnet = evaluate.check_ReLU_alive( net=resnet56, neural_dead_times=neural_dead_times, data_loader=loader, max_data_to_test=10000) def get_statistics(net, relu_list, neural_list, neural_dead_times, sample_num=10000): num_conv = 0 # num of conv layers in the net for mod in net.modules(): if isinstance(mod, torch.nn.modules.conv.Conv2d): num_conv += 1 neural_dead_list = [] #神经元死亡次数的列表 filter_dead_list = [] #卷积核死亡比率的列表 FIRE = [] for i in range(num_conv): for relu_key in list(neural_list.keys()): if relu_list[ i] is relu_key: # find the neural_list_statistics in layer i+1 dead_times = copy.deepcopy(neural_list[relu_key]) neural_dead_list += copy.deepcopy( dead_times).flatten().tolist() neural_num = dead_times.shape[1] * dead_times.shape[ 2] # neural num for one filter # compute sum(dead_times)/(batch_size*neural_num) as label for each filter dead_times = np.sum(dead_times, axis=(1, 2)) FIRE += (dead_times / (neural_num * sample_num)).tolist() # # judge dead filter by neural_dead_times and dead_filter_ratio # dead_times[dead_times < neural_dead_times] = 0 # dead_times[dead_times >= neural_dead_times] = 1 # dead_times = np.sum(dead_times, axis=(1, 2)) # count the number of dead neural for one filter # dead_times = dead_times / neural_num # filter_dead_list+=dead_times.tolist() break active_ratio = 1 - np.array(FIRE) active_filter_list = 1 - np.array(filter_dead_list) neural_activated_list = (sample_num - np.array(neural_dead_list)) / sample_num return neural_activated_list, active_ratio, #active_filter_list nal_vgg, afl_vgg = get_statistics(vgg16, relu_list_vgg, neural_list_vgg, neural_dead_times=neural_dead_times) nal_resnet, afl_resnet = get_statistics( resnet56, relu_list_resnet, neural_list_resnet, neural_dead_times=neural_dead_times) nal_imagenet, afl_imagenet = get_statistics(vgg16_imagenet, relu_list_imagenet, neural_list_imagenet, sample_num=50000, neural_dead_times=40000) # #cdf_of_dead_neurons # plt.figure() # plt.hist([nal_vgg,nal_resnet,nal_imagenet],cumulative=True,histtype='step',bins=1000,density=True,)#linewidth=5.0) #cumulative=False为pdf,true为cdf # # plt.hist(neural_activated_list,cumulative=True,histtype='bar',bins=20,density=True,rwidth=0.6) #cumulative=False为pdf,true为cdf # plt.xlabel('Activation Ratio',fontsize = fontsize) # plt.ylabel('Ratio of Neurons',fontsize = fontsize) # plt.legend(['VGG-16 on CIFAR-10','ResNet-56 on CIFAR-10','VGG-16 on ImageNet'],loc='upper left',fontsize = fontsize) # # plt.savefig('0cdf_of_dead_neurons.jpg') # plt.savefig('cdf_of_dead_neurons.eps',format='eps') # plt.show() # # #cdf_of_inactive_filter # plt.figure() # plt.hist([afl_vgg,afl_resnet,afl_imagenet],cumulative=True,histtype='step',bins=1000,density=True,)#linewidth=5.0) #cumulative=False为pdf,true为cdf # # plt.hist(neural_activated_list,cumulative=True,histtype='bar',bins=20,density=True,rwidth=0.6) #cumulative=False为pdf,true为cdf # plt.xlabel('Activation Ratio',fontsize = fontsize) # plt.ylabel('Ratio of Filters',fontsize = fontsize) # plt.legend(['VGG-16 on CIFAR-10','ResNet-56 on CIFAR-10','VGG-16 on ImageNet'],loc='upper left',fontsize = fontsize) # # plt.savefig('0cdf_of_inactive_filter.jpg') # plt.savefig('cdf_of_inactive_filter.eps',format='eps') # plt.show() #pdf_of_dead_neurons plt.figure() hist_list = [] for nal in [nal_vgg, nal_resnet, nal_imagenet]: hist, bins = np.histogram(nal, bins=[0.1 * i for i in range(11)]) hist_list.append(100 * hist / np.sum(hist)) x_tick = np.array([5, 15, 25, 35, 45, 55, 65, 75, 85, 95]) plt.figure() plt.bar(x_tick - 2, hist_list[0], color='coral', edgecolor='black', label='VGG-16 on CIFAR-10', align='center', width=2) plt.bar(x_tick, hist_list[1], color='cyan', edgecolor='black', label='ResNet-56 on CIFAR-10', align='center', width=2) plt.bar(x_tick + 2, hist_list[2], color='mediumslateblue', edgecolor='black', label='VGG-16 on ImageNet', align='center', width=2) plt.xticks(x_tick, x_tick, size=tick_fontsize) plt.yticks(size=tick_fontsize) plt.xlabel('Activation Ratio (%)', fontsize=label_fontsize) plt.ylabel('% of Neurons', fontsize=label_fontsize) plt.legend(loc='upper right', fontsize=fontsize) # plt.savefig('0pdf_of_dead_neurons.jpg') plt.savefig('pdf_of_dead_neurons.eps', format='eps', bbox_inches='tight') plt.show() #pdf_of_inactive_filter plt.figure() hist_list = [] for active_ratio in [afl_vgg, afl_resnet, afl_imagenet]: hist, bins = np.histogram(active_ratio, bins=[0.1 * i for i in range(11)]) hist_list.append(100 * hist / np.sum(hist)) x_tick = np.array([5, 15, 25, 35, 45, 55, 65, 75, 85, 95]) plt.figure() plt.bar(x_tick - 2, hist_list[0], color='coral', edgecolor='black', label='VGG-16 on CIFAR-10', align='center', width=2) plt.bar(x_tick, hist_list[1], color='cyan', edgecolor='black', label='ResNet-56 on CIFAR-10', align='center', width=2) plt.bar(x_tick + 2, hist_list[2], color='mediumslateblue', edgecolor='black', label='VGG-16 on ImageNet', align='center', width=2) plt.xticks(x_tick, x_tick, size=tick_fontsize) plt.yticks(size=tick_fontsize) plt.xlabel('Activation Ratio (%)', fontsize=label_fontsize) plt.ylabel('% of Filters', fontsize=label_fontsize) plt.legend(loc='upper right', fontsize=fontsize) # plt.savefig('0pdf_of_inactive_filter.jpg') plt.savefig('pdf_of_inactive_filter.eps', format='eps', bbox_inches='tight') plt.show() print()
def speed_up_pruned_net(): fontsize = 15 checkpoint = torch.load( '../data/baseline/vgg16_bn_cifar10,accuracy=0.941.tar') net_original = storage.restore_net(checkpoint) checkpoint = torch.load( '../data/baseline/resnet56_cifar10,accuracy=0.93280.tar') net_original = resnet_cifar.resnet56() net_original.load_state_dict(checkpoint['state_dict']) checkpoint = torch.load( '../data/model_saved/vgg16bn_cifar10_realdata_regressor6_大幅度/checkpoint/flop=39915982,accuracy=0.93200.tar' ) checkpoint = torch.load( '../data/model_saved/resnet56_cifar10_regressor_prunedBaseline2/checkpoint/flop=36145802,accuracy=0.92110.tar' ) net_pruned = storage.restore_net(checkpoint) net_pruned.load_state_dict(checkpoint['state_dict']) # batch_size=[256,512,1024] num_workers = [i for i in range(4, 5)] batch_size = [300, 600, 1000, 1600] device_list = [torch.device('cuda')] # # device_list=[torch.device('cpu')] for num_worker in num_workers: time_original = [] time_pruned = [] for d in device_list: for bs in batch_size: net_original.to(d) net_pruned.to(d) dl = data_loader.create_validation_loader( batch_size=bs, num_workers=num_worker, dataset_name='cifar10') start_time = time.time() evaluate.evaluate_net(net=net_original, data_loader=dl, save_net=False, device=d) end_time = time.time() time_original.append(end_time - start_time) del dl dl = data_loader.create_validation_loader( batch_size=bs, num_workers=num_worker, dataset_name='cifar10') start_time = time.time() evaluate.evaluate_net(net=net_pruned, data_loader=dl, save_net=False, device=d) end_time = time.time() time_pruned.append(end_time - start_time) del dl print('time before pruned:', time_original) print('time after pruned:', time_pruned) acceleration = np.array(time_original) / np.array(time_pruned) baseline = np.ones(shape=len(batch_size)) x_tick = range(len(baseline)) plt.figure() plt.bar(x_tick, acceleration, color='blue', hatch='//') #,label='GPU') # plt.bar(x_tick[len(batch_size):], acceleration[len(batch_size):], color='grey', hatch='\\', label='CPU') # plt.bar(x_tick,baseline,color='red',hatch='*',label='Baseline') plt.xticks(x_tick, batch_size, fontsize=fontsize) plt.yticks(fontsize=fontsize) for x, y in enumerate(list(acceleration)): plt.text(x, y + 0.1, '%.2f x' % y, ha='center', fontsize=fontsize) plt.ylim([0, np.max(acceleration) + 0.5]) plt.xlabel('Batch-Size', fontsize=fontsize) plt.ylabel('Speed-Up', fontsize=fontsize) # plt.legend(loc='upper left') plt.savefig('resnet_gpu_speed_up.eps', format='eps') # plt.savefig(str(num_worker)+'speed_up.jpg') plt.show() print()
projection_matrix = v[:, :dim] return torch.matmul(tensor_2d, projection_matrix) def normalize(tensor): mean = tensor.mean(dim=0) std = tensor.std(dim=0) tensor = (tensor - mean) / std return tensor if __name__ == "__main__": device = torch.device("cuda" if torch.cuda.is_available() else "cpu") # net= vgg.vgg16_bn(pretrained=True).to(device) net1 = resnet_cifar.resnet56().to(device) i = 0 for mod in net1.modules(): if isinstance(mod, torch.nn.Conv2d): i += 1 net2 = resnet.resnet50().to(device) net3 = vgg.vgg16_bn().to(device) test = gcn(in_features=10, out_features=10).to(device) test.forward(net=net1, net_name='resnet56', dataset_name='cifar10', rounds=2)
# def conversion(dataset_name,net_name,checkpoint_path='',checkpoint=None): #转化之前的cifar上的resnet # checkpoint=torch.load(checkpoint_path) # # net=checkpoint.pop('net') # net=resnet_cifar.resnet32() # checkpoint['state_dict']['fc.weight']=checkpoint['state_dict'].pop('linear.weight') # checkpoint['state_dict']['fc.bias']=checkpoint['state_dict'].pop('linear.bias') # # net.load_state_dict(checkpoint['state_dict']) # checkpoint.update(get_net_information(net,dataset_name,net_name)) # torch.save(checkpoint,checkpoint_path) if __name__ == "__main__": # conversion(checkpoint_path='../data/baseline/resnet32_cifar10,accuracy=0.92380.tar',net_name='resnet56',dataset_name='cifar10') checkpoint = torch.load( '/home/victorfang/model_pytorch/data/baseline/resnet56_cifar10,accuracy=0.94230.tar' ) net = resnet_cifar.resnet56() # net.load_state_dict(checkpoint['state_dict']) c = get_net_information(net=net, dataset_name='cifar10', net_name='resnet56') net = restore_net(checkpoint, True) from framework import evaluate, data_loader evaluate.evaluate_net( net, data_loader.create_validation_loader(512, 4, 'cifar10'), False) # c=get_net_information(net=net,dataset_name=dataset_name,net_name='resnet50')