# conv_index=-1 #index of the conv in model.features
# for mod in model.features:
# conv_index+=1
# if isinstance(mod, torch.nn.modules.conv.Conv2d):
# i += 1
# if i == layer_index: # hit the conv to be pruned
# conv=mod
# break
# if percent_of_pruning is not 0:
# if num_to_prune is not 0:
# print('Warning: Param: num_to_prune disabled!')
# num_to_prune=int(conv.out_channels*percent_of_pruning)
# weights = model.features[conv_index].weight.data.cpu().numpy() # get weight of all filters
#
# filter_norm=np.linalg.norm(weights,ord=ord,axis=(2,3)) #compute filters' norm
# if ord==1:
# filter_norm=np.sum(filter_norm,axis=1)
# elif ord==2:
# filter_norm=np.square(filter_norm)
# filter_norm=np.sum(filter_norm,axis=1)
# filter_min_norm_index=np.argsort(filter_norm)
# model=prune_conv_layer_vgg(model,layer_index,filter_min_norm_index[:num_to_prune])
#
# return model
if __name__ == "__main__":
model= vgg.vgg16_bn(pretrained=True)
# select_and_prune_filter(model,layer_index=3,num_to_prune=2,ord=2)
# prune_conv_layer_vgg(model,layer_index=3,filter_index=1)
if pretrained:
kwargs['init_weights'] = False
model = VGG_mask(make_layers(cfg['E']), **kwargs)
if pretrained:
model.load_state_dict(model_zoo.load_url(model_urls['vgg19']))
return model
def vgg19_bn(pretrained=False, **kwargs):
"""VGG_mask 19-layer model (configuration 'E') with batch normalization
Args:
pretrained (bool): If True, returns a model pre-trained on ImageNet
"""
if pretrained:
kwargs['init_weights'] = False
model = VGG_mask(make_layers(cfg['E'], batch_norm=True), **kwargs)
if pretrained:
model.load_state_dict(model_zoo.load_url(model_urls['vgg19_bn']))
return model
if __name__ == "__main__":
from network import vgg
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
net=vgg.vgg16_bn(dataset_name='cifar10').to(device)
checkpoint=torch.load('../data/baseline/vgg16_bn_cifar10,accuracy=0.941.tar')
net.load_state_dict(checkpoint['state_dict'])
reform_net(net)
from framework import evaluate,data_loader
# evaluate.evaluate_net(net=net,data_loader=data_loader.create_validation_loader(batch_size=512,num_workers=4,dataset_name='cifar10'),save_net=False)
net.features[0].prune([0,2,3])
restore_forward(child)
modify_forward(model)
# forward过程中对全局的变量count_ops进行更新
model.eval()
model.forward(data)
restore_forward(model)
if print_flop:
print('flop_num:{}'.format(count_ops))
count_ops_temp=count_ops
count_ops=0
del model
return count_ops_temp
if __name__ == '__main__':
net = vgg.vgg16_bn(pretrained=True)
net.classifier = nn.Sequential(
nn.Dropout(),
nn.Linear(512, 512),
nn.ReLU(True),
nn.Dropout(),
nn.Linear(512, 512),
nn.ReLU(True),
nn.Linear(512, 10),
)
for m in net.modules():
if isinstance(m, nn.Linear):
nn.init.normal_(m.weight, 0, 0.01)
nn.init.constant_(m.bias, 0)
net = net.to(torch.device("cuda" if torch.cuda.is_available() else "cpu"))
def plot_dead_neuron_filter_number(
neural_dead_times=8000,
dataset_name='cifar10',
):
fontsize = 17
label_fontsize = 24
tick_fontsize = 20
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
checkpoint = torch.load(
'../data/baseline/vgg16_bn_cifar10,accuracy=0.941.tar')
vgg16 = storage.restore_net(checkpoint)
vgg16.load_state_dict(checkpoint['state_dict'])
checkpoint = torch.load(
'../data/baseline/resnet56_cifar10,accuracy=0.94230.tar')
resnet56 = resnet_cifar.resnet56().to(device)
resnet56.load_state_dict(checkpoint['state_dict'])
vgg16_imagenet = vgg.vgg16_bn(pretrained=True).to(device)
checkpoint = torch.load(
'/home/victorfang/Desktop/vgg16_bn_imagenet_deadReLU.tar')
relu_list_imagenet = checkpoint['relu_list']
neural_list_imagenet = checkpoint['neural_list']
loader = data_loader.create_validation_loader(batch_size=100,
num_workers=1,
dataset_name=dataset_name)
# loader=data_loader.create_validation_loader(batch_size=1000,num_workers=8,dataset_name='cifar10_trainset')
relu_list_vgg, neural_list_vgg = evaluate.check_ReLU_alive(
net=vgg16,
neural_dead_times=neural_dead_times,
data_loader=loader,
max_data_to_test=10000)
relu_list_resnet, neural_list_resnet = evaluate.check_ReLU_alive(
net=resnet56,
neural_dead_times=neural_dead_times,
data_loader=loader,
max_data_to_test=10000)
def get_statistics(net,
relu_list,
neural_list,
neural_dead_times,
sample_num=10000):
num_conv = 0 # num of conv layers in the net
for mod in net.modules():
if isinstance(mod, torch.nn.modules.conv.Conv2d):
num_conv += 1
neural_dead_list = [] #神经元死亡次数的列表
filter_dead_list = [] #卷积核死亡比率的列表
FIRE = []
for i in range(num_conv):
for relu_key in list(neural_list.keys()):
if relu_list[
i] is relu_key: # find the neural_list_statistics in layer i+1
dead_times = copy.deepcopy(neural_list[relu_key])
neural_dead_list += copy.deepcopy(
dead_times).flatten().tolist()
neural_num = dead_times.shape[1] * dead_times.shape[
2] # neural num for one filter
# compute sum(dead_times)/(batch_size*neural_num) as label for each filter
dead_times = np.sum(dead_times, axis=(1, 2))
FIRE += (dead_times / (neural_num * sample_num)).tolist()
# # judge dead filter by neural_dead_times and dead_filter_ratio
# dead_times[dead_times < neural_dead_times] = 0
# dead_times[dead_times >= neural_dead_times] = 1
# dead_times = np.sum(dead_times, axis=(1, 2)) # count the number of dead neural for one filter
# dead_times = dead_times / neural_num
# filter_dead_list+=dead_times.tolist()
break
active_ratio = 1 - np.array(FIRE)
active_filter_list = 1 - np.array(filter_dead_list)
neural_activated_list = (sample_num -
np.array(neural_dead_list)) / sample_num
return neural_activated_list, active_ratio, #active_filter_list
nal_vgg, afl_vgg = get_statistics(vgg16,
relu_list_vgg,
neural_list_vgg,
neural_dead_times=neural_dead_times)
nal_resnet, afl_resnet = get_statistics(
resnet56,
relu_list_resnet,
neural_list_resnet,
neural_dead_times=neural_dead_times)
nal_imagenet, afl_imagenet = get_statistics(vgg16_imagenet,
relu_list_imagenet,
neural_list_imagenet,
sample_num=50000,
neural_dead_times=40000)
# #cdf_of_dead_neurons
# plt.figure()
# plt.hist([nal_vgg,nal_resnet,nal_imagenet],cumulative=True,histtype='step',bins=1000,density=True,)#linewidth=5.0) #cumulative=False为pdf，true为cdf
# # plt.hist(neural_activated_list,cumulative=True,histtype='bar',bins=20,density=True,rwidth=0.6) #cumulative=False为pdf，true为cdf
# plt.xlabel('Activation Ratio',fontsize = fontsize)
# plt.ylabel('Ratio of Neurons',fontsize = fontsize)
# plt.legend(['VGG-16 on CIFAR-10','ResNet-56 on CIFAR-10','VGG-16 on ImageNet'],loc='upper left',fontsize = fontsize)
# # plt.savefig('0cdf_of_dead_neurons.jpg')
# plt.savefig('cdf_of_dead_neurons.eps',format='eps')
# plt.show()
#
# #cdf_of_inactive_filter
# plt.figure()
# plt.hist([afl_vgg,afl_resnet,afl_imagenet],cumulative=True,histtype='step',bins=1000,density=True,)#linewidth=5.0) #cumulative=False为pdf，true为cdf
# # plt.hist(neural_activated_list,cumulative=True,histtype='bar',bins=20,density=True,rwidth=0.6) #cumulative=False为pdf，true为cdf
# plt.xlabel('Activation Ratio',fontsize = fontsize)
# plt.ylabel('Ratio of Filters',fontsize = fontsize)
# plt.legend(['VGG-16 on CIFAR-10','ResNet-56 on CIFAR-10','VGG-16 on ImageNet'],loc='upper left',fontsize = fontsize)
# # plt.savefig('0cdf_of_inactive_filter.jpg')
# plt.savefig('cdf_of_inactive_filter.eps',format='eps')
# plt.show()
#pdf_of_dead_neurons
plt.figure()
hist_list = []
for nal in [nal_vgg, nal_resnet, nal_imagenet]:
hist, bins = np.histogram(nal, bins=[0.1 * i for i in range(11)])
hist_list.append(100 * hist / np.sum(hist))
x_tick = np.array([5, 15, 25, 35, 45, 55, 65, 75, 85, 95])
plt.figure()
plt.bar(x_tick - 2,
hist_list[0],
color='coral',
edgecolor='black',
label='VGG-16 on CIFAR-10',
align='center',
width=2)
plt.bar(x_tick,
hist_list[1],
color='cyan',
edgecolor='black',
label='ResNet-56 on CIFAR-10',
align='center',
width=2)
plt.bar(x_tick + 2,
hist_list[2],
color='mediumslateblue',
edgecolor='black',
label='VGG-16 on ImageNet',
align='center',
width=2)
plt.xticks(x_tick, x_tick, size=tick_fontsize)
plt.yticks(size=tick_fontsize)
plt.xlabel('Activation Ratio (%)', fontsize=label_fontsize)
plt.ylabel('% of Neurons', fontsize=label_fontsize)
plt.legend(loc='upper right', fontsize=fontsize)
# plt.savefig('0pdf_of_dead_neurons.jpg')
plt.savefig('pdf_of_dead_neurons.eps', format='eps', bbox_inches='tight')
plt.show()
#pdf_of_inactive_filter
plt.figure()
hist_list = []
for active_ratio in [afl_vgg, afl_resnet, afl_imagenet]:
hist, bins = np.histogram(active_ratio,
bins=[0.1 * i for i in range(11)])
hist_list.append(100 * hist / np.sum(hist))
x_tick = np.array([5, 15, 25, 35, 45, 55, 65, 75, 85, 95])
plt.figure()
plt.bar(x_tick - 2,
hist_list[0],
color='coral',
edgecolor='black',
label='VGG-16 on CIFAR-10',
align='center',
width=2)
plt.bar(x_tick,
hist_list[1],
color='cyan',
edgecolor='black',
label='ResNet-56 on CIFAR-10',
align='center',
width=2)
plt.bar(x_tick + 2,
hist_list[2],
color='mediumslateblue',
edgecolor='black',
label='VGG-16 on ImageNet',
align='center',
width=2)
plt.xticks(x_tick, x_tick, size=tick_fontsize)
plt.yticks(size=tick_fontsize)
plt.xlabel('Activation Ratio (%)', fontsize=label_fontsize)
plt.ylabel('% of Filters', fontsize=label_fontsize)
plt.legend(loc='upper right', fontsize=fontsize)
# plt.savefig('0pdf_of_inactive_filter.jpg')
plt.savefig('pdf_of_inactive_filter.eps',
format='eps',
bbox_inches='tight')
plt.show()
print()
return tensor
if __name__ == "__main__":
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
# net= vgg.vgg16_bn(pretrained=True).to(device)
net1 = resnet_cifar.resnet56().to(device)
i = 0
for mod in net1.modules():
if isinstance(mod, torch.nn.Conv2d):
i += 1
net2 = resnet.resnet50().to(device)
net3 = vgg.vgg16_bn().to(device)
test = gcn(in_features=10, out_features=10).to(device)
test.forward(net=net1,
net_name='resnet56',
dataset_name='cifar10',
rounds=2)
c = test.forward(net=net2,
rounds=2,
net_name='resnet50',
dataset_name='imagenet')
print()
# for name, module in network.named_modules():
# if isinstance(module,torch.nn.Conv2d):
x = self.net(x)
x = x.view(x.size(0), -1)
x = F.relu(self.fc1(x))
x = F.dropout(x, training=self.training)
x = self.fc2(x)
return x, F.log_softmax(x)
model = Net()
model = nn.DataParallel(model)
elif args.network == 'Alexnet':
model = alexnet.AlexNet()
model = nn.DataParallel(model)
elif args.network == 'Vgg':
model = vgg.vgg16_bn()
#print(model)
elif args.network == 'VggLReLU':
model = vgg.vgg16_lReLU()
elif args.network == 'VggSigmoid':
model = vgg.vgg16_Sigmoid()
elif args.network == 'VggTanh':
model = vgg.vgg16_Tanh()
elif args.network == 'Vgg_pretrain':
model = models.vgg16(pretrained=True)
elif args.network == 'Vgg_no_pretrain':
model = models.vgg16()
elif args.network == 'Resnet34':
model = resnet.ResNet34()
elif args.network == 'Resnet50':
model = resnet.ResNet50()