def bulid_Net(image, reuse=tf.AUTO_REUSE):
image = tf.reshape(image, [-1, 28, 28, 1])
with tf.variable_scope(name_or_scope='LeNet', reuse=reuse):
arg_scope = LeNet.lenet_arg_scope()
with slim.arg_scope(arg_scope):
logits, end_point = LeNet.lenet(image, 10, is_training=True, dropout_keep_prob=keep_prop)
probs = tf.nn.softmax(logits) # probabilities
return logits, probs, end_point
def train(self):
strides = self.spinBox.value()
kernal_size = self.spinBox_2.value()
filters = self.spinBox_3.value()
print("%d %d %d" % (strides, kernal_size, filters))
self.model2 = LeNet.LeNetBuild(kernal_size, strides, filters)
self.dialog()
self.openWin()
def set_data(self):
self.initialized = False
mnist = input_data.read_data_sets("MNIST_data", reshape=False)
self.x_train, self.y_train = mnist.train.images, mnist.train.labels
self.x_validation, self.y_validation = mnist.validation.images, mnist.validation.labels
self.x_test, self.y_test = mnist.test.images, mnist.test.labels
assert len(self.x_train) == len(self.y_train)
assert len(self.x_validation) == len(self.y_validation)
assert len(self.x_test) == len(self.y_test)
self.image_batch.append(None)
self.image_batch += self.x_train[0].shape
with tf.variable_scope(self.variable_scope_base):
self.x_tensor = tf.placeholder(tf.float32, self.image_batch)
self.y_tensor = tf.placeholder(tf.int32)
self.keep_prob_tensor = tf.placeholder(tf.float32)
self.y_one_hot_tensor = tf.one_hot(indices=tf.cast(self.y_tensor, tf.int32), depth=10)
self.lenet = LeNet.build_lenet(x=self.x_tensor, keep_prob=self.keep_prob_tensor)
self.initialized = True
def train(mnist):
x_train, y_train = mnist.train.images, mnist.train.labels
x_validation, y_validation = mnist.validation.images, mnist.validation.labels
# print(x_train[0].shape)
x_train = np.pad(x_train, ((0, 0), (2, 2), (2, 2), (0, 0)),'constant')
x_validation = np.pad(x_validation, ((0, 0), (2, 2), (2, 2), (0, 0)),'constant')
# print(x_train[0].shape)
x = tf.placeholder(tf.float32, shape=[None, 32, 32, 1])
y = tf.placeholder(tf.int32, shape=[None, ])
one_hot_y = tf.one_hot(y, 10)
y_ = LeNet.LeNet(x)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(labels=one_hot_y, logits=y_)
cross_entropy_mean = tf.reduce_mean(cross_entropy)
train_step = tf.train.GradientDescentOptimizer(RATE).minimize(cross_entropy_mean)
correct_prediction = tf.equal(tf.argmax(y_, 1), tf.argmax(one_hot_y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
with tf.Session() as session:
session.run(tf.global_variables_initializer())
for i in range(EPOCHS):
x_train, y_train = shuffle(x_train, y_train)
for offset in range(0, len(x_train),BATCH_SIZE):
end = offset + BATCH_SIZE
batch_x, batch_y = x_train[offset:end], y_train[offset:end]
session.run(train_step, feed_dict={x:batch_x, y:batch_y})
print("EPOCHS:", i+1)
accuracy_score = session.run(accuracy, feed_dict={x:x_validation, y:y_validation})
print('Validation Accuracy', accuracy_score)
# test
x_test, y_test = mnist.test.images, mnist.test.labels
x_test = np.pad(x_test, ((0, 0), (2, 2), (2, 2), (0, 0)),'constant')
test_accuracy = session.run(accuracy, feed_dict={x:x_test, y:y_test})
print('Test Accuracy', test_accuracy) # test_accuracy = 0.9876
from LeNet import *
from dataset import *
import torch.optim as optim
import torchvision
import torchvision.transforms as transforms
import matplotlib.pyplot as plt
import numpy as np
classes = ('0', '1', '2', '3', '4', '5', '6', '7', '8', '9')
net = LeNet()
criterion = nn.CrossEntropyLoss()
# create your optimizer
optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)
for epoch in range(2): # loop over the dataset multiple times
running_loss = 0.0
for i, data in enumerate(trainloader, 0):
# get the inputs
inputs, labels = data
# wrap them in Variable
inputs, labels = Variable(inputs), Variable(labels)
# in your training loop:
optimizer.zero_grad() # zero the gradient buffers
output = net(inputs)
loss = criterion(output, labels)
loss.backward()
optimizer.step() # Does the update
#torch.cuda.set_device(device_id) # use gpu
if args.enable_lat:
real_model_path = args.model_path + "lat_param.pkl"
print('loading the LAT model')
else:
real_model_path = args.model_path + "naive_param.pkl"
print('loading the naive model')
'''
if args.test_flag:
args.enable_lat = False
'''
# switch models
if args.model == 'lenet':
cnn = LeNet(enable_lat=args.enable_lat,
epsilon=args.epsilon,
pro_num=args.pro_num,
batch_size=args.batchsize,
batch_norm=args.batchnorm,
if_dropout=args.dropout)
elif args.model == 'resnet':
cnn = ResNet50(enable_lat=args.enable_lat,
epsilon=args.epsilon,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
#cnn.apply(conv_init)
elif args.model == 'vgg':
cnn = VGG16(enable_lat=args.enable_lat,
epsilon=args.epsilon,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
def CPdecompose_and_retrain_lenet(learning_rate=0.001,
checkpoints_path="checkpoints_prune_conv"):
tl.set_backend('pytorch')
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print('using: ', end='')
print(torch.cuda.get_device_name(torch.cuda.current_device()))
'''加载模型'''
net = LeNet.lenet5().to(device) # 相当于调用vgg16_bn()函数
# highest_accuracy=get_highestAccuracy_From_File(fileName="accuracy_prune.txt")
# global_step,net=get_globalStep_From_File(net,fileName="globalStep_prune.txt",check_point_path=checkpoints_path,h_accuracy="_98")
global_step, net = get_globalStep_From_File(
net,
check_point_path=checkpoints_path,
stateDict="stateDict_39113_90.pth")
'''-----------------------------------CP分解-----------------------------------'''
N = len(net.features._modules.keys())
for i, key in enumerate(net.features._modules.keys()):
if i >= N - 2:
break
if isinstance(net.features._modules[key],
torch.nn.modules.conv.Conv2d): # 如果是Conv层
conv_layer = net.features._modules[key] # 获取Conv层
rank = max(
conv_layer.weight.data.cpu().numpy().shape) // 3 # 计算rank
decomposed = cp_decomposition_conv_layer(conv_layer, rank) # CP分解
net.features._modules[key] = decomposed
for param in net.parameters():
param.requires_grad = True
net.cuda()
optimizer = optim.SGD(net.parameters(), lr=learning_rate) # CP分解，使用较小的学习率
'''-----------------------------------测试CP分解后的模型在test_set上的准确率-----------------------------------'''
with torch.no_grad():
correct = 0
total = 0
for data in testloader:
net.eval()
images, labels = data
images, labels = images.to(device), labels.to(device)
outputs = net(images)
_, predicted = torch.max(outputs.data, 1) # 取得分最高的那个类
total += labels.size(0)
correct += (predicted == labels).sum()
correct = float(correct.cpu().numpy().tolist())
accuracy = float(correct) / total
print("首次测试 Accuracy :", accuracy)
'''-----------------------------------重训练，每次迭代完测试准确率-----------------------------------'''
for epoch in range(100):
print("Epoch: ", epoch + 1)
sum_loss = 0.0
for i, data in enumerate(trainloader):
net.train()
net.zero_grad()
optimizer.zero_grad() # 梯度清零
inputs, labels = data
inputs, labels = inputs.to(device), labels.to(device)
criterion = torch.nn.CrossEntropyLoss()
loss = criterion(net(inputs), Variable(labels))
loss.backward()
optimizer.step()
sum_loss += loss.item()
if i % 100 == 99: # 每训练100个batch就打印loss
print('[%d, %d] loss: %.03f' %
(epoch + 1, i + 1, sum_loss / 100))
sum_loss = 0.0
with torch.no_grad():
correct = 0
total = 0
for data in testloader:
net.eval()
images, labels = data
images, labels = images.to(device), labels.to(device)
outputs = net(images)
_, predicted = torch.max(outputs.data, 1) # 取得分最高的那个类
total += labels.size(0)
correct += (predicted == labels).sum()
correct = float(correct.cpu().numpy().tolist())
accuracy = float(correct) / total
print("Accuracy :", accuracy)
torch.save(net, 'decomposed_finetuned_model.pth')
train_x, train_y, valid_x, valid_y = load_data.split_data(train_x, train_y)
train_x, train_y = load_data.shared_data(train_x, train_y)
valid_x, valid_y = load_data.shared_data(valid_x, valid_y)
X = T.matrix('input', dtype=theano.config.floatX)
y = T.ivector('labels')
index = T.lscalar('index')
batch_size = 20
learning_rate = 0.01
train_batches = train_x.get_value(borrow=True).shape[0] // batch_size
valid_batches = valid_x.get_value(borrow=True).shape[0] // batch_size
rng = np.random.RandomState(1234)
# create model
layer0_input = X.reshape((batch_size, 1, 28, 28))
conv_net = LeNet.LeNet(input=layer0_input,
batch_size=batch_size,
rng=rng,
n_kernels=[4, 6])
[cost, acc, updates] = conv_net.cost_updates(y, learning_rate=learning_rate)
# gererate model
givens_train = {
X: train_x[index * batch_size:(index + 1) * batch_size],
y: train_y[index * batch_size:(index + 1) * batch_size]
}
givens_valid = {
X: valid_x[index * batch_size:(index + 1) * batch_size],
y: valid_y[index * batch_size:(index + 1) * batch_size]
}
givens = [givens_train, givens_valid]
function = generate_function.function([index], [cost, acc], updates, givens)
[train_model, valid_model] = function.model
graph = tf.Graph()
with graph.as_default():
#1) First we put the input data in a tensorflow friendly form.
tf_train_dataset = tf.placeholder(tf.float32,
shape=(batch_size, image_width,
image_height, image_depth))
tf_train_labels = tf.placeholder(tf.float32,
shape=(batch_size, num_labels))
tf_test_dataset = tf.constant(test_dataset, tf.float32)
tf_inference_data = tf.placeholder(tf.float32,
shape=(1, image_width, image_height,
image_depth))
#2) Then, the weight matrices and bias vectors are initialized
variables = LeNet.variables_lenet5(image_depth=image_depth,
num_labels=num_labels)
#3. The model used to calculate the logits (predicted labels)
model = LeNet.model_lenet5
logits = model(tf_train_dataset, variables)
logits = logits["logits"]
#4. then we compute the softmax cross entropy between the logits and the (actual) labels
# 先求出样本的取平均值 Computes softmax cross entropy between `logits` and `labels`
# 第一步 先对网络最后一层的输出做一个softmax，这一步通常是求取输出属于某一类的概率
# 第二步 第二步是softmax的输出向量和样本的实际标签做一个交叉熵
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=logits,
labels=tf_train_labels))
#5. The optimizer is used to calculate the gradients of the loss function
# 梯度下降优化器
temp = old_weights * mask
new_weights[:] = temp[:] # 获得一个剪枝后的矩阵
if torch.cuda.is_available():
new_conv.cuda()
model.features = torch.nn.Sequential(
*(replace_layers(mod, [old_conv], [new_conv])
for mod in model.features))
return model, mask
if __name__ == "__main__":
# model= vgg.vgg16_bn(pretrained=False)
model = LeNet.lenet5().to("cuda")
# print(model)
model, mask_dict = select_and_prune_conv(model,
layer_list=[1, 2],
sparsity=50)
# helper.print_net_weights(model,conv_layer_index=[1,2],fc_layer_index=[])
for i in mask_dict:
print(mask_dict[i])
# helper.print_net_weights(model,conv_layer_index=[],fc_layer_index=[1,2])
# select_and_prune_filter(model,layer_index=3,num_to_prune=2,ord=2)
# prune_conv_layer(model,layer_index=1,filter_index=[0,1,2])
# model,_=prune_fc_layer(model,fc_layer_index=2,sparsity=50)
# print(model.classifier[0].weight.data.cpu().numpy())
# model,mask_dict=select_and_prune_fc(model, layer_list=[1,2], sparsity=50)
# helper.print_net_weights(model,fc_layer_index=[1,2],conv_layer_index=[])
# select_and_prune_filter(model,layer_index=1,num_to_prune=2,ord=2)
print(inference_acc[0])
max = 0.3 # 设置概率达到80以上才认为
index = 0
for value in inference_acc[0]:
if value > max:
max = value
maxat = index
index = index + 1
return maxat
session = tf.Session()
#2) Then, the weight matrices and bias vectors are initialized
data = tf.placeholder(tf.float32,
shape=(1, image_width, image_height, image_depth))
variables = LeNet.variables_lenet5(image_depth=image_depth,
num_labels=num_labels)
model = LeNet.model_lenet5(data, variables)
inference = LeNet.inference(model["logits"])
# 单个预测
saver = tf.train.Saver()
saver.restore(session, "./lenet5.ckpt")
def inference_click():
img = canvas1.image1
#print(img)
img = img.convert('L')
img = img.resize([image_width, image_height], Image.ANTIALIAS)
# plt.imshow(np.array(img,dtype=np.uint8))
# plt.show()
def CNN_main(train_data,test_data,result_path,train_labels,test_labels,num_classes,epoch,batch_size_factor,optimizer,CNN_model,train_CNN,feature_extraction,feature_extractor_parameters):
'''
Function Description: This function is the main function for the CNN modules and it controls them all,
it decides which method to be used , CNN as a classifer or as a feature extractor and for each one of
them which CNN model and architetcture to be used and also the paramteres for each one of the case is
defined by the variables as described later.
Function Parameters:
-------------------------------------------------------
Train_data_file_name:The name of the train data file , this will be differ depeneding on the CNN model
used,For the following models 'ResNet50','inception_model,'DenseNet121','VGG_pretrained' a resized data
to diminssion 224*224*3 will be used , this one was resized from the original data and was saved after,
as this conversion takes alot of time so as to decrease the computation time.
Test_file_name:The name of the test data file and there are manily two files as mentioned in the previous
variable.
results_directory:the directory name where you would like the output files to be generated
train_data_file_name:This is used to load the training data labels , also you should choose the
right file as there are different training data files due to differnet augmentation methods used
and they differ in their number so the lables will differ also
test_data_file_name: the name of the test labels file , this will also be the same as long as the
test data is not changable.
num_classes:the number of the classes the our data will be classifed to.
epoch:the number of iter through all the training dataset that the CNN model will go through
batch_size_factor:this will take the batch as a factor of the training data size and since the
trainig data in our case is small , then it will be alawys =1
optimizer:this will be the optimizer used for training the CNN model, the variable should have
the name of one of the optimizer in the code , else error will occur.
CNN_model:this define which CNN model to be trained on the training dataset or it will be None if a saved
model will be used as feature extractor
train_CNN: This selects one of the two modes ,=0 means to train a CNN_model or =1 to use a pretrained model
or a saved model to extrat features from the training data directly.
feature_extraction:This is variable wil be used only if train_CNN =0 and it will select between two options;
=0 to trian the CNN model only without using it for feature extraction and if it is =1 the model after
being trained it will be saved and used for feature extraction.
feature_extractor_parameters:
This is a dict of four variables and they are used as an input parameters for the feature extraction
functionan they are as the following :
'CNN_model': this will be the CNN model used for feature extraction, it may takes different type , it
may be string , in the case of pretrained model so the string will be equal to the name of the selected model
or it may take 'all' in this case all the pretrained models will be used , in case of saved model this can take
two forms, the first to be string if the feature_extraction=1 , this mean that a saved model will be used directly
without being trained and if feature_extraction=1 it will be th model that have just been trained and saved.
'model_type':to choose between the pretrained models on imagenet dataset or the saved trained models
on the training set.
'classifer_name': this is the classifer name you would like to use , it can be 'all', or one of them.
'hyperprametres': this is another dict in which the classifer hyperparmaters is defined, it contain four main
parameters , the value of the neigbors to look to for the KNN classifer and the for SVC classifer thera are
two hyperparamters 'penalty' and 'C' and for the fully conncted classifer there are four hyperparamters
'dropouts','activation_functions','opt','epoch'.If any of this classifers is not used then it's hyperparmater should
be =None.
'''
opt=optimizers.choosing(optimizer)
if train_CNN==1:
if CNN_model=='simple_model':
simple_model.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
elif CNN_model=='LeNet':
LeNet.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
elif CNN_model=='AlexNet':
AlexNet.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
elif CNN_model=='ZFNet':
ZFNet.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
elif CNN_model=='VGG':
VGG.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
elif CNN_model=='VGG_pretrained':
VGG_pretrained.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
elif CNN_model=='ResNet50':
ResNet50.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
elif CNN_model=='inception_model':
InceptionResNetV2.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
elif CNN_model== 'DenseNet121':
DenseNet121.model(train_data,train_labels,test_data,test_labels,opt,epoch,batch_size_factor,num_classes,result_path,feature_extraction,feature_extractor_parameters)
else:
print('Value Error: CNN_model took unexpected value')
sys.exit()
elif train_CNN==0:
CNN_feature_extractor.CNN_feature_extraction_classsification(feature_extractor_parameters,result_path)
else:
print('Value Error: train_CNN took unexpected value')
sys.exit()
def get_cloest_char(img):
dtext = r'/|\=-'
pixs = np.sum(img > 0)
return dtext[LeNet.demo(net, [Image.fromarray(img)])] if pixs > 5 else ' '
import cv2
import numpy as np
from PIL import Image
import LeNet
imgpath = r'./fbk.jpg'
net = LeNet.load_build_net('./weights/LeNet_epoch49.pth')
def get_cloest_char(img):
dtext = r'/|\=-'
pixs = np.sum(img > 0)
return dtext[LeNet.demo(net, [Image.fromarray(img)])] if pixs > 5 else ' '
def close_demo(image, size=(5, 5)):
kernel = cv2.getStructuringElement(cv2.MORPH_RECT, size)
binary = cv2.morphologyEx(image,
cv2.MORPH_CLOSE,
kernel,
anchor=(-1, -1),
iterations=3)
return binary
image = cv2.imread(imgpath)
#image = cv2.resize(image, (600,400), interpolation = cv2.INTER_AREA)
canny = cv2.Canny(image, 100, 230)
#canny = close_demo(canny,(3,3))
#canny = cv2.resize(canny, (400,600), interpolation = cv2.INTER_AREA)
cv2.imshow('canny', canny)
def choose_model():
# switch models
print(args.model)
print(args.layerlist)
if args.model == 'lenet':
cnn = LeNet(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
batch_norm=args.batchnorm,
if_dropout=args.dropout)
elif args.model == 'resnet':
cnn = ResNet50(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'resnet18':
cnn = ResNet18(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg16':
cnn = VGG16(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg11':
cnn = VGG11(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg13':
cnn = VGG13(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'vgg19':
cnn = VGG19(enable_lat=args.enable_lat,
layerlist=args.layerlist,
epsilon=args.epsilon,
alpha=args.alpha,
pro_num=args.pro_num,
batch_size=args.batchsize,
if_dropout=args.dropout)
elif args.model == 'densenet':
cnn = DenseNet()
cnn.cuda()
if args.enable_lat:
cnn.choose_layer()
return cnn
aw_bits = 8
acc_bits = 32
#network
network_name = "lenet"
report_fname = "data_{}_CIFAR10_postTrainMasking_new.txt".format(network_name)
config_fname = "{}.mc".format(network_name)
if (network_name == "vgg11bn"):
import models.cifar10.vgg_cifar as vgg
network = vgg.vgg11_bn_cifar("./data/ref_model")
checkpoint_name = "{}_CIFAR10_bestAccuracy_9240.pt".format(network_name)
elif (network_name == "lenet"):
import LeNet
checkpoint_name = "{}_CIFAR10_bestAccuracy_7528.pt".format(network_name)
network = LeNet.LeNet()
elif (network_name == "resnet32"):
import distiller.models.cifar10 as models
checkpoint_name = "{}_CIFAR10_bestAccuracy_9358.pt".format(network_name)
network = models.resnet_cifar.resnet32_cifar()
else:
errorString = "No checkpoint for {} network. Provide training".format(
network_name)
raise ValueError(network_name)
#load dataset
dataset = "CIFAR10"
batch_size = 100
if dataset == "CIFAR10":
#load test set
transform_test = transforms.Compose([
def prune_lenet_and_train(learning_rate=conf.learning_rate,
fc_layer_list=[1, 2],
conv_layer_list=[1, 2],
sparsity=90):
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
print('using: ', end='')
print(torch.cuda.get_device_name(torch.cuda.current_device()))
'''定义模型'''
net = LeNet.lenet5().to(device) # 相当于调用vgg16_bn()函数
highest_accuracy = get_highestAccuracy_From_File()
global_step, net = get_globalStep_From_File(net,
fileName="global_step.txt")
# '''剪枝Conv层，生成新的模型'''
# num_conv=0 # 用于计算卷积层的数目（LeNet中有2个卷积层）
# for mod in net.features:
# if isinstance(mod, torch.nn.modules.conv.Conv2d):
# num_conv+=1
# for i in range(1,num_conv+1):
# net = select_and_prune_filter(net,
# layer_index=i, # 对第i个卷积层进行剪枝
# percent_of_pruning=percent_of_pruning, # 剪枝率
# ord=ord)# 范数
'''稀疏化Conv层，生成新的模型'''
net, conv_mask_dict = select_and_prune_conv(net,
layer_list=conv_layer_list,
sparsity=sparsity)
'''稀疏化FC层，生成新的模型'''
# net,fc_mask_dict=select_and_prune_fc(net,layer_list=fc_layer_list,sparsity=sparsity) # LeNet5有3个FC层，只能对前两个剪枝
'''定义损失函数'''
criterion = nn.CrossEntropyLoss() # 交叉熵
'''定义优化器'''
optimizer = optim.SGD(
net.parameters(),
lr=learning_rate) # 优化方式为mini-batch momentum-SGD，并采用L2正则化（权重衰减）
'''准备数据(将所有输入都统一为224*224)'''
transform = transforms.ToTensor()
trainset = datasets.MNIST(root='./data/',
train=True,
download=True,
transform=transform) # 定义训练数据集
train_loader = torch.utils.data.DataLoader(trainset,
batch_size=BATCH_SIZE,
shuffle=True) # 定义训练批处理数据
testset = datasets.MNIST(root='./data/',
train=False,
download=True,
transform=transform) # 定义测试数据集
validation_loader = torch.utils.data.DataLoader(testset,
batch_size=BATCH_SIZE,
shuffle=False) # 定义测试批处理数据
print('{} 剪枝完毕，测试准确率 '.format(datetime.now())) # 无之前保存的模型
highest_accuracy, global_step = evaluate_model(
net,
validation_loader,
highest_accuracy=highest_accuracy,
global_step=global_step) # 评估模型
# 注意，highest_accuracy为剪枝后当前的最高准确率，不是全局（不包含剪枝前）
save_prune_result(net,
global_step=global_step,
sparsity=sparsity,
accuracy=highest_accuracy)
print("{} Start Retraining LeNet5...".format(datetime.now()))
for epoch in range(0, NUM_EPOCH):
sum_loss = 0
print("{} Epoch number: {}".format(datetime.now(), epoch + 1))
net.train() # 设为训练模式
for step, data in enumerate(train_loader, 0):
net.train()
images, labels = data
images, labels = images.to(device), labels.to(device)
optimizer.zero_grad()
'''这里是retrain，因此需要对Conv层和FC层做一些改动'''
net = replace_conv_weights(net, conv_mask_dict=conv_mask_dict)
# net=replace_fc_weights(net, fc_mask_dict=fc_mask_dict)
# helper.print_net_weights(net,conv_layer_index=[1,2],fc_layer_index=[])
outputs = net(images)
loss = criterion(outputs, labels)
loss.backward()
optimizer.step()
global_step += 1
sum_loss += loss.item()
if step % 100 == 99: # 每训练100个batch就打印loss
print('[%d, %d] loss: %.03f' %
(epoch + 1, step + 1, sum_loss / 100))
sum_loss = 0.0
print("{} Start validation, global step = {}".format(
datetime.now(), global_step))
with torch.no_grad(): # 每运行完一个epoch就进行验证（Validation）
correct = 0
total = 0
for data in validation_loader:
net.eval()
images, labels = data
images, labels = images.to(device), labels.to(device)
outputs = net(images)
_, predicted = torch.max(outputs.data, 1) # 取得分最高的那个类
total += labels.size(0)
correct += (predicted == labels).sum()
correct = float(correct.cpu().numpy().tolist())
accuracy = correct / total
print("{} 验证集准确率 Accuracy = {:.6f}".format(datetime.now(),
accuracy))
if accuracy > highest_accuracy: # 如果有更高的准确率，则保存模型
highest_accuracy = accuracy
save_prune_result(net,
global_step=global_step,
sparsity=sparsity,
accuracy=highest_accuracy)
if __name__ == '__main__':
#Train or not
is_train = False
show = ToPILImage()
#Set Parameters
num_class = 10
batch_size = 64
lr = 0.02
num_epochs = 5
#Data preprocessing
[loader_train,
loader_test] = DataUtils.DataPreprocess1(batch_size=batch_size)
#Chose Model
model = LeNet.LeNet()
model = model.cuda(device=0)
#Chose Criterion and Optimizer
criterion = nn.CrossEntropyLoss()
optimizer = optim.SGD(model.parameters(), lr=lr)
#Train Stage
if is_train:
epoch_count = 0
model.train(mode=True)
while epoch_count < num_epochs:
for data in loader_train:
img, label = data
#Convert label to one_hot format
#importing the libraries like keras, sklearn and matplotlib
from keras.optimizers import SGD
from keras.preprocessing.image import ImageDataGenerator
import matplotlib.pyplot as plt
import LeNet
from sklearn.metrics import classification_report
# Loading images from the directory in batches of 64
traingen=ImageDataGenerator(rescale=1./255,rotation_range=30,horizontal_flip=True,fill_mode='nearest')
testgen=ImageDataGenerator(rescale=1./255)
train_generator=traingen.flow_from_directory(r'C:\Users\91945\Downloads\Analysis\DL.ai\Internship\T2\data\train set',class_mode='categorical',batch_size=64,target_size=(128,128),shuffle=True)
test_generator=testgen.flow_from_directory(r'C:\Users\91945\Downloads\Analysis\DL.ai\Internship\T2\data\val set',class_mode='categorical',batch_size=64,target_size=(128,128),shuffle=True)
# Building my CNN model
lenet=LeNet.LeNet(128,128,3,2)
model1=lenet.build()
model1.summary()
# Fine tuning the optimizer
opt=SGD(lr=0.01,decay=1e-5,momentum=0.9)
# Compiling my CNN model according to my data
model1.compile(loss="binary_crossentropy",optimizer=opt,metrics=["accuracy"])
H1=model1.fit_generator(train_generator,epochs=30,validation_data=test_generator)
pred=model1.predict(test_generator)
model1.save("model1.hdf5")
# Plotting the accuracy loss graph with respect to epochs
plt.style.use("ggplot")
hist=H1.history