def main():
class_names = cifar10.load_class_names()
images_train, cls_idx_train, labels_train = cifar10.load_training_data()
images_test, cls_idx_test, labels_test = cifar10.load_test_data()
#Plot the first 9 training images and labels
plot_9images(images=images_train[0:9], cls_idx_true=cls_idx_train[0:9],
all_cls_names=class_names, smooth=True)
# Build your predictor
w1, b1, w2, b2 = train(images_train, labels_train, images_test, cls_idx_test)
# Visualize your prediction
print('--------------------------------Neural Network--------------------------------')
samples = random.sample(range(len(images_test)), 9)
plot_9images(images=images_test[samples], cls_idx_true=cls_idx_test[samples],
cls_idx_pred=predict(images_test[samples], w1, b1, w2, b2), all_cls_names=class_names, smooth=True)
print(f'\nAccuracy: {(predict(images_test, w1, b1, w2, b2) == cls_idx_test).mean() * 100}%\n')
knn_idx = kNN(images_train, cls_idx_train, images_test)
print('-------------------------------k-Nearest Neighbor-------------------------------')
samples = random.sample(range(len(images_test)), 9)
plot_9images(images=images_test[samples], cls_idx_true=cls_idx_test[samples],
cls_idx_pred=knn_idx[samples], all_cls_names=class_names, smooth=True)
print(f'\nAccuracy: {(knn_idx == cls_idx_test).mean() * 100}%\n')
def load_cifar10():
# make directory if not exist
if not os.path.isdir("data"):
os.mkdir("data")
if not os.path.isdir("data/CIFAR-10"):
os.mkdir("data/CIFAR-10")
# download and extract if not done yet
# data is downloaded from data_url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
# to data_path = "data/CIFAR-10/"
cifar10.data_path = "data/CIFAR-10/"
cifar10.maybe_download_and_extract()
# load data
x_train, y_train_cls, y_train = cifar10.load_training_data()
x_test, y_test_cls, y_test = cifar10.load_test_data()
class_names = cifar10.load_class_names()
x_train = x_train.astype(np.float32)
y_train_cls = y_train_cls.astype(np.int32)
y_train = y_train.astype(np.float32)
x_test = x_test.astype(np.float32)
y_test_cls = y_test_cls.astype(np.int32)
y_test = y_test.astype(np.float32)
data = (x_train, y_train_cls, y_train, x_test, y_test_cls, y_test,
class_names)
return data
def load_data():
################ download dataset ####################
cifar10.maybe_download_and_extract()
################ load train and test data ####################
images_train, _, _ = cifar10.load_training_data()
images_test, _, _ = cifar10.load_test_data()
return images_train, images_test
def download_dataset():
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
print("Size of:")
print("- Training-set:\t\t{}".format(len(images_train)))
print("- Test-set:\t\t{}".format(len(images_test)))
return class_names, images_train, cls_train, labels_train, images_test, cls_test, labels_test
def _load_cifar10(self):
import cifar10
cifar10.data_path = self.cifar10_path
print('Load {} dataset...'.format(self.dataset_name))
# The CIFAR-10 data-set is about 163 MB and will be downloaded automatically if it is not
# located in the given path.
cifar10.maybe_download_and_extract()
self.train_data, self.train_label, _ = cifar10.load_training_data()
self.num_trains = self.train_data.shape[0]
self.test_data, self.test_label, _ = cifar10.load_test_data()
print('Load {} dataset SUCCESS!'.format(self.dataset_name))
def train():
class_names = cifar10.load_class_names()
print(class_names)
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
with open(r'E:\tmp\CIFAR-10\inception_cifar10_test.pkl', mode='rb') as file:
transfer_values_test = pickle.load(file)
with open(r'E:\tmp\CIFAR-10\inception_cifar10_train.pkl', mode='rb') as file:
transfer_values_train = pickle.load(file)
print(transfer_values_train.shape)
print(type(transfer_values_train))
model=tl_Inception(r'E:\tmp\tl_inception')
model.fit(transfer_values_train,labels_train,transfer_values_test,labels_test)
def main():
class_names = cifar10.load_class_names()
images_train, cls_idx_train, labels_train = cifar10.load_training_data()
images_test, cls_idx_test, labels_test = cifar10.load_test_data()
# Plot the first 9 training images and labels
plot_9images(images=images_train[0:9], cls_idx_true=cls_idx_train[0:9],
all_cls_names=class_names, smooth=True)
# Build your predictor
train(images_train, labels_train)
# Visualize your prediction
samples = random.sample(range(len(images_test)), 9)
plot_9images(images=images_test[samples], cls_idx_true=cls_idx_test[samples],
cls_idx_pred=predict(images_test[samples]), all_cls_names=class_names, smooth=True)
def simple_shortcut_test():
cifar10.maybe_download_and_extract()
images_test, cls_test, labels_test = cifar10.load_test_data()
images_test.astype(np.float32)
model_params = {
"conv1": [3,3, 64],
"conv2": [3,3, 64],
"conv3": [3,3,128],
"conv4": [3,3, 28],
}
with tf.variable_scope("test"):
x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x')
with tf.variable_scope("test"):
x = tf.placeholder(tf.float32, shape=[None, 32, 32, 3], name='x')
net1 = convolution_layer(x, model_params["conv1"], [1,1,1,1],name="conv1")
net1 = convolution_layer(net1, model_params["conv2"], [1,1,1,1],name="conv2")
net2 = convolution_layer(net1, model_params["conv4"], [1,2,2,1],name="conv3")
net2 = convolution_layer(net2, model_params["conv3"], [1,1,1,1],name="conv4")
net = shortcut(net2, net1, "s1")
net_avg = global_avg_pooling(net)
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
netout = sess.run([net, net_avg],feed_dict={x:images_test[0:5,:,:,:]})
return netout
def test(test_image_num=10):
test_images, test_cls, test_labels = cifar10.load_test_data()
number_of_batch = 100
with tf.Session() as thread:
model = tf.train.import_meta_graph(
os.path.join(os.getcwd(), "model.meta"))
model.restore(thread, os.path.join(os.getcwd(), "model"))
accuracy = []
for i in range(test_image_num):
accuracy.append(thread.run(number_accurate_prediction,feed_dict={X:test_images[i*number_of_batch:(i+1)*number_of_batch],\
y:test_labels[i*number_of_batch:(i+1)*number_of_batch]}))
print("prediction accuracy of the first testing images: ",
np.array(accuracy) / 100.0)
def load_cifar10_data():
"""Load CIFAR-10 dataset
Returns:
tuple of floats -- img, label for train, valid, test
"""
import cifar10
cifar10.maybe_download_and_extract()
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
x_train = images_train[:45000]
y_train = labels_train[:45000]
x_validate = images_train[45000:]
y_validate = labels_train[45000:]
x_test = images_test
y_test = labels_test
return x_train, y_train, x_validate, y_validate, x_test, y_test
def main(unused_argv):
training_data, _, training_label = cifar10.load_training_data()
test_data, _, test_label = cifar10.load_test_data()
print(test_label)
cifar_classifier = tf.estimator.Estimator(model_fn=cnn_model_fn,
model_dir='./model')
tensors_to_log = {"probabilities": "softmax_tensor"}
logging_hook = tf.train.LoggingTensorHook(tensors=tensors_to_log,
every_n_iter=100)
train_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": training_data},
y=training_label,
batch_size=batch_size,
num_epochs=None,
shuffle=True)
cifar_classifier.train(input_fn=train_input_fn, steps=steps)
eval_input_fn = tf.estimator.inputs.numpy_input_fn(x={"x": test_data},
y=test_label,
num_epochs=1,
shuffle=False)
test_result = cifar_classifier.evaluate(input_fn=eval_input_fn)
print(test_result)
def get_data(data_type='mnist', is_training=True):
if data_type == 'mnist':
raw_data = input_data.read_data_sets('./data/mnist/', one_hot=True)
shape = [28, 28, 1]
if is_training:
size = len(raw_data.train.images)
images = np.reshape(raw_data.train.images, [size] + shape)
labels = raw_data.train.labels
else:
size = len(raw_data.test.images)
images = np.reshape(raw_data.test.images, [size] + shape)
labels = raw_data.test.labels
elif data_type == 'cifar10':
if is_training:
images, _, labels = cifar10.load_training_data()
else:
images, _, labels = cifar10.load_test_data()
else:
raise Exception('data type error: {}'.format(data_type))
datasource = Datasource(images, labels)
return datasource
def test():
class_names = cifar10.load_class_names()
test_batch_size = 64
# images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
with open(r'E:\tmp\CIFAR-10\inception_cifar10_test.pkl', mode='rb') as file:
transfer_values_test = pickle.load(file)
# with open(r'E:\tmp\CIFAR-10\inception_cifar10_train.pkl', mode='rb') as file:
# transfer_values_train = pickle.load(file)
model = tl_Inception(r'E:\tmp\tl_inception')
score_list = []
_X = transfer_values_test
_Y = labels_test
_im = images_test
for i in range(1):
_x,_y = random_batch(_X,_Y,test_batch_size)
pred = model.predict(_x)
print(pred)
true_cls = np.argmax(_y,axis=1)
print(true_cls)
print('score is ', 1-np.mean(pred != true_cls))
print('wrong classified samples ',np.sum(pred != true_cls))
score_list.append(1-np.mean(pred != true_cls))
print('mean score is ',np.mean(score_list))
#test with plot
im_list = np.random.choice(10000,size=10,replace=False)
im = _im[im_list]
label = np.argmax(_Y[im_list],axis=1)
_x = _X[im_list]
pred = model.predict(_x)
for i in range(10):
print(im[i].shape)
plot_helper(im[i])
plot_helper(_x[i])
print(label[i],'-',pred[i])
print(label[i],'',class_names[label[i]],'-',pred[i],class_names[pred[i]])
def train(images, one_hot_labels):
import cifar10
def data_reshape(data, type_img=True):
if type_img:
return data.reshape(-1, 3072,1)
else:
return data.reshape(-1, 10, 1)
images_train, cls_idx_train, labels_train = cifar10.load_training_data()
images_train = data_reshape(images_train)
labels_train = data_reshape(labels_train, type_img=False)
training_data = [(x, y) for x, y in zip(images_train, cls_idx_train)]
images_test, cls_idx_test, lables_test = cifar10.load_test_data()
images_test = data_reshape(images_test)
lables_test = data_reshape(lables_test, type_img=False)
test_data = [(x, y) for x, y in zip(images_test, cls_idx_test)]
net = nn.nnetwork([3072, 120, 10])
net.train(training_data, 10, 40, 3.0, test_data=test_data)
def main():
args = parse_args()
model_name = args.model_name
model_path = args.model_path.split(".")[0]
# 加载测试数据集
images_test, _, labels_test = load_test_data()
print("images_test.shape = ", images_test.shape)
# 构建模型（模型也可以直接从.meta文件中恢复）
X = tf.placeholder(tf.float32, [None, 32, 32, 3])
Y_ = tf.placeholder(tf.float32, [None, 10])
keep_prob = tf.placeholder(tf.float32)
if model_name == "cifar10-5layers":
_, Y = cifar10_5layers(X, keep_prob)
elif model_name == "cifar10-8layers":
_, Y = cifar10_8layers(X, keep_prob)
else:
print("wrong model name!")
print("model name: 'cifar10-5layers' or 'cifar10-8layers'")
raise KeyError
correct_prediction = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
with tf.Session() as sess:
# 恢复模型权重
saver.restore(sess, model_path)
# 测试，注意测试阶段keep_prob=1
print(
"test accuracy = ",
sess.run(accuracy,
feed_dict={
X: images_test,
Y_: labels_test,
keep_prob: 1
}))
def main():
args = get_arguments()
allvars = get_all_model_variables(args)
# Load Test Dataset
if (allvars['model2load'] == 'fcnn') or (allvars['model2load'] == 'lenet'):
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets(allvars['data_dir'], one_hot=True)
X = mnist.test.images
y = mnist.test.labels
# Free Memory
mnist = None
if (allvars['model2load'] == 'nin') or (allvars['model2load'] == 'densenet'):
import cifar10
cifar10.data_path = allvars['data_dir']
cifar10.maybe_download_and_extract()
X, _, y = cifar10.load_test_data()
# Free Memory
cifar = None
X, y = pre_process_data(X, y, allvars['model2load'])
X, y = collect_correctly_predicted_images(X, y, allvars)
eps_rescale = np.max(np.abs( np.max(X.flatten()) - np.min(X.flatten()) ))
deepfool_norms, deepfooled_vec = get_deepfool_ellenorms(X, y, allvars)
print('DeepFool fooling ratio: '+str(np.mean(deepfooled_vec)*100)+' %')
print('DeepFool mean epsilon: '+str(np.mean(deepfool_norms[deepfooled_vec==1])/eps_rescale))
if args.rho_mode:
print('Computing performance metrics...')
print()
rho_1 = get_robustness_metrics(X, y, deepfool_norms, deepfooled_vec, allvars, method='rho1')
rho_2 = get_robustness_metrics(X, y, deepfool_norms, deepfooled_vec, allvars, method='rho2')/eps_rescale
eps99 = get_robustness_metrics(X, y, deepfool_norms, deepfooled_vec, allvars, method='eps99')/eps_rescale
print('rho_1 = ' + str(rho_1))
print('rho_2 = ' + str(rho_2))
print('eps99 = ' + str(eps99))
np.savetxt(allvars['output_dir'] + 'robustness_' + allvars['model2load'] + '_' + str(allvars['n_images']) + \
'_' + str(int(allvars['max_epsilon'] * 1000)) + '.csv',
np.array([rho_1, rho_2, eps99]),
delimiter=";")
else:
print('Computing fooling ratios...')
print()
epsilon = np.array(np.linspace(0.001, allvars['max_epsilon'], 10))
fool_dict = {'epsilon': 0, 'FGS': 1, 'Alg1': 2, 'Alg2': 3, 'rand': 4, 'DeepFool': 5}
fool_mtx = np.zeros([len(epsilon), len(fool_dict)])
for i in range(len(epsilon)):
eps = epsilon[i]*eps_rescale
print(allvars['model2load'] + ': realization '+str(i+1)+'/'+str(len(epsilon))+'...' )
fool_mtx[i, fool_dict['epsilon']] = epsilon[i]
fool_mtx[i, fool_dict['FGS']] = get_foolratio(X, y, eps, allvars, method='FGS')
fool_mtx[i, fool_dict['Alg1']] = get_foolratio(X, y, eps, allvars, method='Alg1')
fool_mtx[i, fool_dict['Alg2']] = get_foolratio(X, y, eps, allvars, method='Alg2')
fool_mtx[i, fool_dict['rand']] = get_foolratio(X, y, eps, allvars, method='rand')
fool_mtx[i, fool_dict['DeepFool']] = np.mean(np.array((deepfool_norms<eps) * (deepfooled_vec==1)))
np.savetxt(allvars['output_dir']+'fool_summary_' + allvars['model2load'] +'_' +str(allvars['n_images'])+\
'_'+str(int(allvars['max_epsilon']*1000))+'.csv', fool_mtx, delimiter=";")
save_foolratio_fig(fool_mtx,
allvars['figs_dir'] + 'fig_'+ allvars['model2load'] +'_' +str(allvars['n_images'])+\
'_'+str(int(allvars['max_epsilon']*1000))+'.eps',
fool_dict, legend=True)
import tflearn from tflearn.data_utils import shuffle, to_categorical from tflearn.layers.core import input_data, dropout, fully_connected from tflearn.layers.conv import conv_2d, max_pool_2d from tflearn.layers.estimator import regression from tflearn.data_preprocessing import ImagePreprocessing from tflearn.data_augmentation import ImageAugmentation import scipy import tensorflow as tf # Data loading and preprocessing import cifar10 cifar10.data_path = "data/CIFAR-10/" X, Y, labels_train = cifar10.load_training_data() X_test, Y_test, labels_test = cifar10.load_test_data() #(X, Y), (X_test, Y_test) = cifar10.load_data() X, Y = shuffle(X, Y) Y = to_categorical(Y, 10) Y_test = to_categorical(Y_test, 10) # Real-time data preprocessing img_prep = ImagePreprocessing() img_prep.add_featurewise_zero_center() img_prep.add_featurewise_stdnorm() # Real-time data augmentation img_aug = ImageAugmentation() img_aug.add_random_flip_leftright() img_aug.add_random_rotation(max_angle=25.)
def main(argv = None):
if (argv is None):
argv = sys.argv
try:
try:
opts = argv
TRAIN = 1
pretrain = 0
for item in opts:
print (item)
opt = item[0]
val = item[1]
if (opt == '-t'):
TRAIN = val
if (opt == '-q_bits'):
q_bits = val
if (opt == '-parent_dir'):
parent_dir = val
if (opt == '-base_model'):
base_model = val
if (opt == '-pretrain'):
pretrain = val
print('pretrain is {}'.format(pretrain))
except getopt.error, msg:
raise Usage(msg)
NUM_CLASSES = 10
dropout = 0.8 # probability of keep
BATCH_SIZE = 128
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.001
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average.
DISPLAY_FREQ = 20
TEST = 0
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
DOCKER = 0
mask_dir = parent_dir
model_dir = parent_dir
PREV_MODEL_EXIST = 1
(weights_mask,biases_mask)= initialize_weights_mask(0, mask_dir + 'masks/' + 'base.pkl' )
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
t_data = format_data(images_train, labels_train)
test_data = format_data(images_test, labels_test)
DATA_CNT = len(images_train)
NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE
training_data_list = []
weights, biases = initialize_variables(PREV_MODEL_EXIST, parent_dir, q_bits, pretrain)
weights, biases = compute_weights_nbits(weights, biases, q_bits)
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, NUM_CLASSES])
if (TRAIN == 1):
TRAIN_OR_TEST = 1
else:
TRAIN_OR_TEST = 0
keep_prob = tf.placeholder(tf.float32)
images = pre_process(x, TRAIN_OR_TEST)
# images = pre_process(x, 1)
pred = cov_network(images, weights, biases, keep_prob)
# print(pred)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)
loss_value = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
global_step = tf.contrib.framework.get_or_create_global_step()
num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE
decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY)
# Decay the learning rate exponentially based on the number of steps.
lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
global_step,
decay_steps,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
opt = tf.train.GradientDescentOptimizer(lr)
grads = opt.compute_gradients(loss_value)
org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads]
new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask)
# Apply gradients.
train_step = opt.apply_gradients(new_grads, global_step=global_step)
init = tf.global_variables_initializer()
accuracy_list = np.zeros(30)
accuracy_list = np.zeros(5)
# Launch the graph
print('Graph launching ..')
with tf.Session() as sess:
sess.run(init)
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
for key in keys:
sess.run(weights[key].assign(weights[key].eval()*weights_mask[key]))
# sess.run(biases[key].assign(biases[key].eval()*biases_mask[key]))
print('pre train pruning info')
prune_info(weights, 0)
print(78*'-')
start = time.time()
if TRAIN == 1:
for i in range(0,10000):
(batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE)
train_acc, cross_en = sess.run([accuracy, loss_value], feed_dict = {
x: batch_x,
y: batch_y,
keep_prob: 1.0})
if (i % DISPLAY_FREQ == 0):
# prune_info(weights, 0)
print('This is the {}th iteration, time is {}'.format(
i,
time.time() - start
))
print("accuracy is {} and cross entropy is {}".format(
train_acc,
cross_en
))
# accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:29]))
accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:4]))
if (np.mean(accuracy_list) > 0.8):
print("training accuracy is large, show the list: {}".format(accuracy_list))
NUMBER_OF_BATCH = 10000 / BATCH_SIZE
t_acc = []
for i in range(0,NUMBER_OF_BATCH):
(batch_x, batch_y) = test_data.feed_next_batch(BATCH_SIZE)
test_acc = sess.run(accuracy, feed_dict = {
x: batch_x,
y: batch_y,
keep_prob: 1.0})
t_acc.append(test_acc)
print("test accuracy is {}".format(t_acc))
test_acc = np.mean(t_acc)
accuracy_list = np.zeros(5)
print('test accuracy is {}'.format(test_acc))
if (test_acc > 0.823):
print('Exiting the training, test accuracy is {}'.format(test_acc))
break
_ = sess.run(train_step, feed_dict = {
x: batch_x,
y: batch_y,
keep_prob: dropout})
if (TRAIN == 1):
keys = ['cov1','cov2','fc1','fc2','fc3']
weights_save = {}
centroids_save = {}
for key in keys:
centroids_save[key] = centroids_var[key].eval()
weights_save[key] = weights[key].eval()
with open(parent_dir + 'weights/'+ 'weights'+str(q_bits)+'.pkl','wb') as f:
pickle.dump((weights_save, biases_orgs, cluster_index,centroids_save),f)
NUMBER_OF_BATCH = 10000 / BATCH_SIZE
t_acc = []
for i in range(0,NUMBER_OF_BATCH):
(batch_x, batch_y) = test_data.feed_next_batch(BATCH_SIZE)
test_acc = sess.run(accuracy, feed_dict = {
x: batch_x,
y: batch_y,
keep_prob: 1.0})
t_acc.append(test_acc)
print("test accuracy is {}".format(t_acc))
# save_pkl_model(weights, biases, model_name)
return np.mean(t_acc)
def main(argv=None):
if (argv is None):
argv = sys.argv
try:
try:
opts = argv
first_time_load = True
parent_dir = './'
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
prune_thresholds = {}
WITH_BIASES = False
next_iter_save = False
for key in keys:
prune_thresholds[key] = 0.
for item in opts:
print(item)
opt = item[0]
val = item[1]
if (opt == '-pcov1'):
prune_thresholds['cov1'] = val
if (opt == '-pcov2'):
prune_thresholds['cov2'] = val
if (opt == '-pfc1'):
prune_thresholds['fc1'] = val
if (opt == '-pfc2'):
prune_thresholds['fc2'] = val
if (opt == '-pfc3'):
prune_thresholds['fc3'] = val
if (opt == '-first_time'):
first_time_load = val
if (opt == '-init_file_name'):
init_file_name = val
if (opt == '-train'):
TRAIN = val
if (opt == '-prune'):
PRUNE = val
if (opt == '-parent_dir'):
parent_dir = val
if (opt == '-lr'):
lr = val
if (opt == '-with_biases'):
WITH_BIASES = val
if (opt == '-cRates'):
cRates = val
if (opt == '-iter_cnt'):
iter_cnt = val
if (opt == '-save'):
next_iter_save = val
if (opt == '-org_file_name'):
org_file_name = val
print('pruning thresholds are {}'.format(prune_thresholds))
except getopt.error, msg:
raise Usage(msg)
NUM_CLASSES = 10
dropout = 0.5
BATCH_SIZE = 128
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.001
INITIAL_LEARNING_RATE = lr
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average.
DISPLAY_FREQ = 50
TEST = 1
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
DOCKER = 0
if (DOCKER == 1):
base_model_name = '/root/20170206.pkl'
model_name = '/root/pruning'
mask_dir = '/root/mask'
else:
mask_dir = parent_dir
weights_dir = parent_dir
# model_name = 'test.pkl'
# model_name = '../tf_official_docker/tmp.pkl'
if (next_iter_save):
file_name_part = org_file_name
else:
file_name_part = compute_file_name(cRates)
(weights_mask, biases_mask) = initialize_weights_mask(
first_time_load, mask_dir, 'mask_crate' + file_name_part + '.pkl')
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
if (TRAIN):
images_train, cls_train, labels_train = cifar10.load_training_data(
)
images_test, cls_test, labels_test = cifar10.load_test_data()
t_data = training_data(images_train, labels_train)
DATA_CNT = len(images_train)
NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE
else:
if (PRUNE):
images_test, cls_test, labels_test = cifar10.load_test_data()
else:
images_test, cls_test, labels_test = cifar10.load_test_data()
training_data_list = []
if (first_time_load):
PREV_MODEL_EXIST = 0
weights, biases = initialize_variables(PREV_MODEL_EXIST, '')
else:
PREV_MODEL_EXIST = 1
weights_dir = parent_dir
if (next_iter_save):
file_name_part = org_file_name
else:
file_name_part = compute_file_name(cRates)
weights, biases = initialize_variables(
PREV_MODEL_EXIST,
parent_dir + 'weight_crate' + file_name_part + '.pkl')
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, NUM_CLASSES])
keep_prob = tf.placeholder(tf.float32)
images = pre_process(x, TRAIN)
no_preprocesss_images = pre_process(x, False)
weights_new = {}
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
for key in keys:
weights_new[key] = weights[key] * tf.constant(weights_mask[key],
dtype=tf.float32)
pred = cov_network(images, weights_new, biases, keep_prob)
test_pred = cov_network(no_preprocesss_images, weights_new, biases,
keep_prob)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=pred,
labels=y)
loss_value = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(test_pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
opt = tf.train.AdamOptimizer(lr)
grads = opt.compute_gradients(loss_value)
org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads]
# new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask)
# Apply gradients.
train_step = opt.apply_gradients(org_grads)
init = tf.global_variables_initializer()
accuracy_list = np.zeros(20)
# accuracy_list = np.zeros(5)
train_acc_list = []
# Launch the graph
print('Graph launching ..')
# config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = (0.7)
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333)
with tf.Session() as sess:
sess.run(init)
print('pre train pruning info')
prune_info(weights_new, 0)
print(78 * '-')
start = time.time()
if TRAIN == 1:
# for i in range(0,60000):
# for i in range(0,6):
for i in range(0, 20000):
(batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE)
train_acc, cross_en = sess.run([accuracy, loss_value],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
if (i % DISPLAY_FREQ == 0):
print(
'This is the {}th iteration of {}pruning, time is {}'
.format(i, cRates,
time.time() - start))
print("accuracy is {} and cross entropy is {}".format(
train_acc, cross_en))
# accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:49]))
accuracy_list = np.concatenate(
(np.array([train_acc]), accuracy_list[0:19]))
# accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:4]))
if (i % (DISPLAY_FREQ * 50) == 0 and i != 0):
train_acc_list.append(train_acc)
file_name_part = compute_file_name(cRates)
file_name = 'weight_crate' + str(
file_name_part) + '.pkl'
save_pkl_model(weights, biases, parent_dir,
file_name)
print("saved the network")
if (np.mean(accuracy_list) > 0.81
and train_acc >= 0.82):
prune_info(weights_new, 0)
print(accuracy_list)
accuracy_list = np.zeros(20)
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
print(test_acc)
if (test_acc > 0.823):
print(
"training accuracy is large, show the list: {}"
.format(accuracy_list))
break
_ = sess.run(train_step,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: dropout
})
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
print("test accuracy is {}".format(test_acc))
if (next_iter_save):
print('saving for the next iteration of dynamic surgery')
file_name_part = compute_file_name(cRates)
file_name = 'weight_crate' + file_name_part + '.pkl'
save_pkl_model(weights, biases, parent_dir, file_name)
file_name_part = compute_file_name(cRates)
with open(parent_dir + 'mask_crate' + file_name_part + '.pkl',
'wb') as f:
pickle.dump(weights_mask, f)
if (TRAIN):
file_name_part = compute_file_name(cRates)
file_name = 'weight_crate' + file_name_part + '.pkl'
save_pkl_model(weights, biases, parent_dir, file_name)
with open(parent_dir + 'training_data' + file_name + '.pkl',
'wb') as f:
pickle.dump(train_acc_list, f)
if (PRUNE):
print('saving pruned model ...')
file_name_part = compute_file_name(cRates)
prune_weights(prune_thresholds, weights, biases, weights_mask,
cRates, iter_cnt, parent_dir)
file_name = 'weight_crate' + file_name_part + '.pkl'
print('saving pruned network')
save_pkl_model(weights, biases, parent_dir, file_name)
return test_acc
def evaluate(self, test_data):
test_results = [ (np.argmax(self.output(x)), y)
for x, y in test_data ]
return sum(int(x==y) for x, y in test_results)
if __name__ == '__main__':
import cifar10
def data_reshape(data, type_img=True):
if type_img:
return data.reshape(-1, 3072,1)
else:
return data.reshape(-1, 10, 1)
images_train, cls_idx_train, labels_train = cifar10.load_training_data()
images_train = data_reshape(images_train)
labels_train = data_reshape(labels_train, type_img=False)
training_data = [(x, y) for x, y in zip(images_train, cls_idx_train)]
images_test, cls_idx_test, lables_test = cifar10.load_test_data()
images_test = data_reshape(images_test)
lables_test = data_reshape(lables_test, type_img=False)
test_data = [(x, y) for x, y in zip(images_test, cls_idx_test)]
net = nnetwork([3072, 50, 10])
net.train(training_data, 10, 40, 3.0, test_data=test_data)
def main():
"""
# 下载并解压数据集(已下载可忽略)
data_url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz"
data_path = "./data/"
download.maybe_download_and_extract(data_url,data_path)
"""
args = parse_args()
# 导入数据集并显示数据集信息
class_names = load_label_names()
images_train, _, labels_train = load_training_data()
images_test, _, labels_test = load_test_data()
images_train, labels_train, images_valid, labels_valid = \
split_train_data(images_train, labels_train, shuffle=True)
print("classes names:", class_names)
print("shape of training images:", images_train.shape)
print("shape of training labels (one-hot):", labels_train.shape)
print("shape of valid images:", images_valid.shape)
print("shape of valid labels (one-hot):", labels_valid.shape)
print("shape of test images:", images_test.shape)
print("shape of testing labels (one-hot):", labels_test.shape)
# 将数据集分成mini-batches.
images_train_batches, labels_train_batches = create_mini_batches(images_train, \
labels_train, \
shuffle=True)
print("shape of one batch training images:", images_train_batches[0].shape)
print("shape of one batch training labels:", labels_train_batches[0].shape)
print("shape of last batch training images:",
images_train_batches[-1].shape)
print("shape of last batch training labels:",
labels_train_batches[-1].shape)
img_size = images_train.shape[1]
num_channels = images_train.shape[-1]
num_classes = len(class_names)
batch_size = images_train_batches[0].shape[0]
num_batches = len(images_train_batches)
# 创建模型
X = tf.placeholder(tf.float32, [None, img_size, img_size, num_channels])
Y_ = tf.placeholder(tf.float32, [None, num_classes])
keep_prob = tf.placeholder(tf.float32)
init_method = init_methods[args.init_method]
if args.model_name == "cifar10-5layers":
logit, Y = cifar10_5layers(X, keep_prob, init_method)
elif args.model_name == "cifar10-8layers":
logit, Y = cifar10_8layers(X, keep_prob, init_method)
# 交叉熵损失和准确率
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logit,
labels=Y_)
cross_entropy = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(Y, 1), tf.argmax(Y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# 训练过程
tr_step = []
tr_acc = []
tr_loss = []
va_step = []
va_acc = []
va_loss = []
train_steps = 50000
lr = 0.0001
train_step = tf.train.AdamOptimizer(lr).minimize(cross_entropy)
init = tf.global_variables_initializer()
saver = tf.train.Saver()
with tf.Session() as sess:
sess.run(init)
for i in range(train_steps + 1):
# 获取一个mini batch数据
j = i % num_batches # j用于记录第几个mini batch
batch_X = images_train_batches[j]
batch_Y = labels_train_batches[j]
if j == num_batches - 1: # 遍历一遍训练集（1个epoch）
images_train_batches, labels_train_batches = create_mini_batches(images_train, \
labels_train, \
shuffle=True)
# 训练一次
sess.run(fetches=train_step,
feed_dict={
X: batch_X,
Y_: batch_Y,
keep_prob: 0.5
})
# 每100次打印并记录一组训练结果
if i % 100 == 0:
train_accuracy, train_loss = sess.run([accuracy, cross_entropy], \
feed_dict={X: batch_X, Y_: batch_Y, keep_prob: 1})
print("steps =", i, "train loss =", train_loss,
" train accuracy =", train_accuracy)
tr_step.append(i)
tr_acc.append(train_accuracy)
tr_loss.append(train_loss)
# 每500次打印并记录一次测试结果（验证集）
if i % 500 == 0:
valid_accuracy, valid_loss = sess.run([accuracy, cross_entropy], \
feed_dict={X: images_valid, Y_: labels_valid, keep_prob: 1})
va_step.append(i)
va_acc.append(valid_accuracy)
va_loss.append(valid_loss)
print("steps =", i, "validation loss =", valid_loss,
" validation accuracy =", valid_accuracy)
# 每10000次保存一次训练模型到本地
if i % 10000 == 0 and i > 0:
model_name = args.model_name + "_" + args.init_method
model_name = os.path.join("./models", model_name)
saver.save(sess, model_name, global_step=i)
# 保存训练日志到本地
train_log = "train_log_" + args.model_name + "_" + args.init_method + ".txt"
train_log = os.path.join("./results", train_log)
with open(train_log, "w") as f:
f.write("steps\t" + "accuracy\t" + "loss\n")
for i in range(len(tr_step)):
row_data = str(tr_step[i]) + "\t" + str(round(
tr_acc[i], 3)) + "\t" + str(round(tr_loss[i], 3)) + "\n"
f.write(row_data)
valid_log = "valid_log_" + args.model_name + "_" + args.init_method + ".txt"
valid_log = os.path.join("./results", valid_log)
with open(valid_log, "w") as f:
f.write("steps\t" + "accuracy\t" + "loss\n")
for i in range(len(va_step)):
row_data = str(va_step[i]) + "\t" + str(round(
va_acc[i], 3)) + "\t" + str(round(va_loss[i], 3)) + "\n"
f.write(row_data)
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np
import time
from datetime import timedelta
import cifar10
cifar10.download()
print(cifar10.load_class_names())
train_img, train_cls, train_labels = cifar10.load_training_data()
test_img, test_cls, test_labels = cifar10.load_test_data()
print('Training set:', len(train_img), 'Testing set:', len(test_img))
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y_true = tf.placeholder(tf.float32, [None, 10])
def conv_layer(input, size_in, size_out, use_pooling=True):
w = tf.Variable(tf.truncated_normal([3, 3, size_in, size_out], stddev=0.1))
b = tf.Variable(tf.constant(0.1, shape=[size_out]))
conv = tf.nn.conv2d(input, w, strides=[1, 1, 1, 1], padding='SAME')
y = tf.nn.relu(conv + b)
if use_pooling:
y = tf.nn.max_pool(y,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
def train(self):
self.train_vali_graph()
images_train, cls_train, labels_train = cifar10.load_training_data()
images_validation, cls_validation, labels_validation = cifar10.load_validation_data(
)
images_test, cls_test, labels_test = cifar10.load_test_data()
init_op = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init_op)
tr_acc_list = []
val_acc_list = []
iters_list = []
for i in range(max_iter):
x_batch_train, y_batch_labels = self.random_batch(
images_train, labels_train)
_, cost, tr_acc = sess.run(
[self.train_op, self.total_loss, self.train_accuracy],
feed_dict={
self.train_image_placeholder: x_batch_train,
self.train_label_placeholder: y_batch_labels
})
if i % 1000 == 0:
x_batch_vali, y_batch_vali = self.random_batch(
images_validation, labels_validation)
vali_acc = sess.run(
[self.validation_accuracy],
feed_dict={
self.validation_image_placeholder: x_batch_vali,
self.validation_label_placeholder: y_batch_vali
})
print(
'Step %d Loss=%.3f Training Accuracy = %.3f Validation Accuracy = %.3f'
% (i, cost, tr_acc, vali_acc[0]))
tr_acc_list.append(tr_acc)
val_acc_list.append(vali_acc)
iters_list.append(i)
print('Optimization Done!')
start = 0
num_test = len(images_test)
final_acc = []
while start < num_test:
end = min(start + FLAGS.batch_size, num_test)
test_acc = sess.run(
[self.test_accuracy],
feed_dict={
self.validation_image_placeholder:
images_test[start:end, :, :, :],
self.validation_label_placeholder:
labels_test[start:end, :]
})
final_acc.append(test_acc)
start = end
f_acc = np.mean(final_acc)
print('Final test accuracy = %.3f' % f_acc)
fig = plt.figure(figsize=(10, 10))
plt.plot(iters_list, tr_acc_list)
plt.plot(iters_list, val_acc_list)
plt.legend(['training accuracy', 'validation accuracy'],
loc='upper left')
plt.savefig('accuracy.png')
def pre_process(images, img_size_cropped, num_channels, training):
# Use TensorFlow to loop over all the input images and call
# the function above which takes a single image as input.
images = tf.map_fn(lambda image: pre_process_image(image, img_size_cropped, num_channels, training), images)
return images
##
#load data
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
print(class_names)
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
print("Size of:")
print("- Training-set:\t\t{}".format(len(images_train)))
print("- Test-set:\t\t{}".format(len(images_test)))
img_size_cropped = 24
##
X = tf.placeholder(tf.float32, shape=[None, img_size, img_size, num_channels], name='X')
images = pre_process(images=X, img_size_cropped=img_size_cropped, num_channels=num_channels,training=True)
#distorted_images = pre_process(images=images, img_size_cropped=img_size_cropped,num_channels=num_channels,training=True) #no es necesario
y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name='y_true')
y_true_cls = tf.argmax(y_true, dimension=1)
def main():
args = get_arguments()
modelvarnames = get_all_model_variables(args)
# Load Dataset
# X = mnist.train.images
# y = mnist.train.labels
# Xtest = mnist.test.images
# ytest = mnist.test.labels
# mnist = None # Free Memory
if modelvarnames['model2load'] == 'fcnnmnist':
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets(modelvarnames['data_dir'],
one_hot=True)
Xtrain, ytrain = mnist.train.next_batch(modelvarnames['batch-size'])
Xtest, ytest = mnist.test.next_batch(modelvarnames['n_images'])
elif modelvarnames['model2load'] == 'fcnncifar':
from cifar10 import maybe_download_and_extract, load_training_data, load_test_data
maybe_download_and_extract()
Xtrain, class_labels_train, ytrain = load_training_data()
Xtest, class_labels_test, ytest = load_test_data()
else:
print('error loading datasets')
return
# pre-process data
Xtrain, ytrain = pre_process_data(Xtrain, ytrain,
modelvarnames['model2load'])
Xtest, ytest = pre_process_data(Xtest, ytest, modelvarnames['model2load'])
print(ytest)
# Load pre-trained model
model = Model(modelvarnames, restore=False)
# Create necesary directories
if not os.path.exists('./results/'):
os.makedirs('./results/')
# Set up simulation parameters
eps_rescale = np.max(
np.abs(np.max(Xtrain.flatten()) - np.min(Xtrain.flatten())))
epsilon = modelvarnames['min_epsilon'] * eps_rescale
print('Computing fooling ratios...')
print()
iterations = int(5e4 / modelvarnames['batch-size'] *
modelvarnames['epochs'])
time = np.unique(np.linspace(0, iterations, 100 + 1).astype(int))
t = 0
eps_test = modelvarnames['max_epsilon'] * eps_rescale
FLAG_adv_train = False
# Initialize Result matrices
summary_dict = {
'adv-training': 0,
'std-train-error': 1,
'adv-train-error': 2,
'std-test-error': 3,
'adv-test-error': 4
# 'std-min-margin': 5,
# 'adv-min-margin': 6,
# 'std-mean-margin': 7,
# 'adv-mean-margin': 8
}
summary_mtx = np.zeros([len(time), len(summary_dict)])
norm_mtx_list = []
norm_dict = {
'mixed-half-inf': 0,
'mixed-1-inf': 1,
'mixed-1-1': 2,
'fro': 3,
'spectral': 4,
'mixed-1-2': 5,
'mixed-2-1': 6
}
for jj in range(model.nlayers):
norm_mtx_list.append(np.zeros([len(time), len(norm_dict)]))
# Start Simulation
for ii in range(iterations):
if modelvarnames['model2load'] == 'fcnnmnist':
X, y = mnist.train.next_batch(modelvarnames['batch-size'])
X, y = pre_process_data(X, y, modelvarnames['model2load'])
else:
X, y = next_batch(modelvarnames['batch-size'], Xtrain, ytrain)
if FLAG_adv_train: # Add Adv noise only if AdvTraining has started
X = get_adversarial_examples(
X, y, model, epsilon, iterations=modelvarnames['its-advtrain'])
acc_train = model.sess.run(model.acc,
feed_dict={
model.Xtrue: X,
model.Ytrue: y,
model.pkeep: 1.0
})
print('adv-acc = {}'.format(acc_train))
if acc_train >= modelvarnames['adv-acc-thrs']:
if epsilon >= modelvarnames[
'max_epsilon'] * eps_rescale: # max epsilon reached --> increase its
modelvarnames['its-advtrain'] = np.minimum(
modelvarnames['its-advtrain'] + 1, 10)
epsilon *= 1.1
epsilon = np.minimum(
epsilon, modelvarnames['max_epsilon'] * eps_rescale)
print('\n\n eps = {}, its = {} \n\n'.format(
epsilon, modelvarnames['its-advtrain']))
model.sess.run(model.trainop,
feed_dict={
model.Xtrue: X,
model.Ytrue: y,
model.pkeep: 1.0
})
# Run training operation (update the weights)
if ii in time: # Get performance indicators and norms of weights
for jj in range(model.nlayers): # Store norms of weights
layer_norms = LayerNorms(model, layer=jj + 1)
norm_mtx_list[jj][
t,
norm_dict['mixed-half-inf']] = layer_norms.mixed_half_inf
norm_mtx_list[jj][
t, norm_dict['mixed-1-inf']] = layer_norms.mixed_1_inf
norm_mtx_list[jj][
t, norm_dict['mixed-1-1']] = layer_norms.mixed_1_1
norm_mtx_list[jj][t, norm_dict['fro']] = layer_norms.fro
norm_mtx_list[jj][t,
norm_dict['spectral']] = layer_norms.spectral
norm_mtx_list[jj][
t, norm_dict['mixed-1-2']] = layer_norms.mixed_1_2
norm_mtx_list[jj][
t, norm_dict['mixed-2-1']] = layer_norms.mixed_2_1
# Load a batch of images from the TEST set
if modelvarnames['model2load'] == 'fcnnmnist':
X, y = mnist.test.next_batch(modelvarnames['n_images'])
X, y = pre_process_data(X, y, modelvarnames['model2load'])
else:
X, y = next_batch(modelvarnames['n_images'], Xtest, ytest)
# Compute the TEST accuracy WITHOUT adversarial noise
summary_mtx[t,
summary_dict['std-test-error']] = get_err(model, X, y)
# Compute the TEST accuracy WITH adversarial noise
X = get_adversarial_examples(X, y, model, eps_test, iterations=10)
summary_mtx[t,
summary_dict['adv-test-error']] = get_err(model, X, y)
# Load a batch of images from the TRAIN set
if modelvarnames['model2load'] == 'fcnnmnist':
X, y = mnist.train.next_batch(modelvarnames['n_images'])
X, y = pre_process_data(X, y, modelvarnames['model2load'])
else:
X, y = next_batch(modelvarnames['n_images'], Xtrain, ytrain)
# Compute the TRAIN accuracy WITHOUT adversarial noise
summary_mtx[t, summary_dict['std-train-error']] = get_err(
model, X, y)
# Compute the TRAIN accuracy WITH adversarial noise
X = get_adversarial_examples(X, y, model, eps_test, iterations=10)
summary_mtx[t, summary_dict['adv-train-error']] = get_err(
model, X, y)
summary_mtx[t, summary_dict['adv-training']] = int(FLAG_adv_train)
print('{:.0f}% done.. {}'.format(ii / iterations * 100,
summary_mtx[t, :]))
if ii / iterations > modelvarnames[
'start-advtrain']: # We start adv training at 50% of the iterations
FLAG_adv_train = True
# Load a batch of images from the TEST set
t += 1
# Save training/test errors
ID = str(np.random.randint(0, int(1e5)))
np.savetxt('./results/ID-{}_{}-summary.csv'.format(
ID, modelvarnames['model2load']),
summary_mtx,
delimiter=';')
# Save norms of weights
for jj in range(model.nlayers):
np.savetxt('./results/ID-{}_{}-W_{:.0f}.csv'.format(
ID, modelvarnames['model2load'], jj + 1),
norm_mtx_list[jj],
delimiter=';')
if not os.path.exists('./hyperparams/'):
os.makedirs('./hyperparams/')
hyperparams = np.array([
modelvarnames['start-advtrain'], modelvarnames['min_epsilon'],
modelvarnames['max_epsilon'], modelvarnames['batch-size'],
modelvarnames['epochs'], modelvarnames['adv-acc-thrs'],
modelvarnames['lr']
]).astype(float)
np.savetxt('./hyperparams/ID-{}_{}.csv'.format(
ID, modelvarnames['model2load']),
hyperparams,
delimiter=';')
from sklearn.decomposition import PCA
from sklearn.manifold import TSNE
from sklearn.metrics import confusion_matrix
cifar10.data_path = "data/CIFAR-10/"
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
print(class_names)
print('Loading the training set...')
training_images, training_cls_integers, trainig_one_hot_labels = cifar10.load_training_data(
)
print('Loading the test set...')
testing_images, testing_cls_integers, testing_one_hot_labels = cifar10.load_test_data(
)
print("-Number of images in the training set:\t\t{}".format(
len(training_images)))
print("-Number of images in the testing set:\t\t{}".format(
len(testing_images)))
def plot_imgs(imgs, true_class, predicted_class=None):
assert len(imgs) == len(true_class)
# Creating a placeholders for 9 subplots
fig, axes = plt.subplots(3, 3)
# Adjustting spacing.
if predicted_class is None:
#xxx = "Don't run"
xxx = "Run"
num_iterations = 1000000
train_batch_size = 64
imgchosen = 2473 #should be SCr
Numberoftrainingsamples = i
Numberoftestsamples = 5250
require_improvement = 1500
_images_per_file = int(Numberoftrainingsamples/3)
_num_files_train = 3 #5
_num_images_train = _num_files_train * _images_per_file
data_path = "D:/Iheya_n/HvassTutResults/2BatSCrNSe/NewResults/3to1ratio/{}Train/{}/".format(Numberoftrainingsamples,j) #data_path = cifar10.data_path
data_path1 = "D:/Iheya_n/HvassTutResults/2BatSCrNSe/NewResults/3to1ratio/{}Train/".format(Numberoftrainingsamples)
class_names = cifar10.load_class_names(data_path1)
images_train, cls_train, labels_train = cifar10.load_training_data(data_path1,_num_images_train,data_path)
images_test, cls_test, labels_test = cifar10.load_test_data(data_path1,data_path)
openfile = open(data_path + "results.txt", "a")
print("Size of:")
print("- Training-set:\t\t{}".format(len(images_train)))
print("- Test-set:\t\t{}".format(len(images_test)))
openfile.write("size of:\n- Training-set:\t\t{}\n- Test-set:\t\t{}".format(len(images_train),len(images_test)))
#save_dir = data_path + 'checkpoints/'
save_dir = data_path + 'checkpoints/'
if not os.path.exists(save_dir):
os.makedirs(save_dir)
save_path = os.path.join(save_dir, 'cifar10_cnn')
#from cifar10 import num_classes num_classes = 2 ##################################################################################### ##################################################################################### # DATA LOADING # ##################################################################################### ##################################################################################### cifar10.maybe_download_and_extract() class_names_load = cifar10.load_class_names() class_names_load images_train, cls_train_load, labels_train_load = cifar10.load_training_data([b'deer',b'horse']) images_test, cls_test_load, labels_test_load = cifar10.load_test_data([b'dog',b'truck']) images_test2, cls_test_load2, labels_test_load2 = cifar10.load_test_data([b'ship',b'frog']) images_test3, cls_test_load3, labels_test_load3 = cifar10.load_test_data([b'deer',b'horse']) # binarising classes, labels and class names cls_train,labels_train = class_binariser(cls_train_load),label_binariser(labels_train_load) cls_test,labels_test = class_binariser(cls_test_load),label_binariser(labels_test_load) cls_test2,labels_test2 = class_binariser(cls_test_load2),label_binariser(labels_test_load2) # to see if the classifier will work on this other set cls_test3,labels_test3 = class_binariser(cls_test_load3),label_binariser(labels_test_load3) class_names = class_name_binariser(class_names_load,cls_test_load) class_names2 = class_name_binariser(class_names_load,cls_test_load2) class_names3 = class_name_binariser(class_names_load,cls_test_load3)
def train(self, config):
### Optimizer for GAN
d_optim = tf.train.AdamOptimizer(self.Learning_Rate, beta1=config.beta1) \
.minimize(self.d_loss, var_list=self.d_vars)
g_optim = tf.train.AdamOptimizer(self.Learning_Rate, beta1=config.beta1) \
.minimize(self.g_loss, var_list=self.g_vars)
# download the data and extract training and test splits
cifar10.maybe_download_and_extract()
Train_Images, Train_Labels = cifar10.load_training_data()
sio.savemat("./CIFAR-10/Train_Images.mat",
mdict={"Train_Images": Train_Images})
sio.savemat("./CIFAR-10/Train_Labels.mat",
mdict={"Train_Labels": Train_Labels})
Test_Images, Test_Labels = cifar10.load_test_data()
sio.savemat("./CIFAR-10/Test_Images.mat",
mdict={"Test_Images": Test_Images})
sio.savemat("./CIFAR-10/Test_Labels.mat",
mdict={"Test_Labels": Test_Labels})
print("Executing GAN")
tf.global_variables_initializer().run()
sample_z = np.random.uniform(-1, 1, size=(self.sample_num, self.z_dim))
fake_batches = int(self.Generated_labels.shape[0] / self.batch_size)
# initial parameters for the GAN and the fraction of split
gan_iter = 1000
alpha = 0.1
fake_labels = np.copy(self.Generated_labels)
fixed_label = 0
iter_labels = np.arange((fixed_label + 1), 10)
for fold in range(len(iter_labels)):
tf.global_variables_initializer().run()
counter = 0
Gan_data_train, Gan_data_label = self.load_cifar(
seed=seed,
fixed_label=fixed_label,
iter_label=iter_labels[fold],
frac=alpha)
Gan_batches = int(Gan_data_train.shape[0] / self.batch_size)
##### Starting with GAN
start_time = time.time()
for gan_epoch in range(gan_iter):
for idx in xrange(0, Gan_batches):
batch_images = Gan_data_train[idx *
self.batch_size:(idx + 1) *
self.batch_size]
batch_labels = Gan_data_label[idx *
self.batch_size:(idx + 1) *
self.batch_size]
rand_index = np.random.permutation(batch_images.shape[0])
batch_images = batch_images[rand_index]
batch_labels = batch_labels[rand_index]
batch_z = np.random.uniform(-1,
1,
size=(self.sample_num,
self.z_dim))
# Update D network
_ = self.sess.run(
[d_optim],
feed_dict={
self.inputs: batch_images / 127.5 - 1.,
self.z: batch_z,
self.y: batch_labels
})
# Update G network
_ = self.sess.run([g_optim],
feed_dict={
self.z: batch_z,
self.y: batch_labels
})
counter = counter + 1
if np.mod(counter, 100) == 0:
errD_fake = self.d_loss_fake.eval({
self.z: batch_z,
self.y: batch_labels
})
errD_real = self.d_loss_real.eval({
self.inputs:
batch_images / 127.5 - 1.,
self.y:
batch_labels
})
errG = self.g_loss.eval({
self.z: batch_z,
self.y: batch_labels
})
print("Epoch: [%2d] [%4d/%4d] time: %4.4f, d_loss: %.8f, g_loss: %.8f" \
% (gan_epoch, idx, Gan_batches, time.time() - start_time, errD_fake + errD_real, errG))
samples, d_loss, g_loss = self.sess.run(
[self.sampler, self.d_loss, self.g_loss],
feed_dict={
self.z: sample_z,
self.inputs: batch_images / 127.5 - 1.,
self.y: batch_labels
})
save_images(
samples, image_manifold_size(samples.shape[0]),
'./{}/train_{:02d}_{:d}.png'.format(
config.sample_dir, gan_epoch, fold))
##### GAN Iterations Terminates
#### Generate fake Images by utilizing GAN
Generated_Samples = []
for index in range(fake_batches):
sample_z = np.random.uniform(-1,
1,
size=(self.batch_size,
self.z_dim))
sample_labels = fake_labels[index *
self.batch_size:(index + 1) *
self.batch_size]
samples = self.sess.run([self.sampler],
feed_dict={
self.z: sample_z,
self.y: sample_labels
})
Generated_Samples.append(samples)
Generated_Samples = np.reshape(
np.concatenate(Generated_Samples, axis=0), (-1, 32, 32, 3))
### Save Random Fake Images Generated by GAN to check the quality
index_images = np.random.randint(10000, size=100)
random_samples = Generated_Samples[index_images]
save_images(random_samples,
image_manifold_size(random_samples.shape[0]),
'./{}/fake_{:d}.png'.format(config.sample_dir, fold))
##### Inverse Transform the digits to 255
Generated_Samples = np.round(
(Generated_Samples + 1.) * 127.5).astype(np.float)
sio.savemat('./Generated_Data/Images_%d_%d_%d.mat' %
(alpha, fixed_label, iter_labels[fold]),
mdict={'Images': Generated_Samples})
sio.savemat('./Generated_Data/Labels_%d_%d_%d.mat' %
(alpha, fixed_label, iter_labels[fold]),
mdict={'Labels': fake_labels})
from datetime import timedelta import os # Functions and classes for loading and using the Inception model. import inception # We use Pretty Tensor to define the new classifier. import prettytensor as pt import cifar10 from cifar10 import num_classes model = inception.Inception() batch_size = 64 class_names = cifar10.load_class_names() images_train, cls_train, labels_train = cifar10.load_training_data() transfer_values_test, cls_test, labels_test = cifar10.load_test_data(1) # Get the first images from the test-set. # images = images_test[0:9] # # # Get the true classes for those images. # cls_true = cls_test[0:9] # # # Plot the images and labels using our helper-function above. # plot_images(images=images, cls_true=cls_true, smooth=False) transfer_len = model.transfer_len x = tf.placeholder(tf.float32, shape=[None, transfer_len], name='x') y_true = tf.placeholder(tf.float32, shape=[None, num_classes], name='y_true') y_true_cls = tf.argmax(y_true, dimension=1)
def main(argv=None):
if (argv is None):
argv = sys.argv
try:
try:
opts = argv
first_time_load = False
parent_dir = ''
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
prune_thresholds = {}
TRAIN = True
for key in keys:
prune_thresholds[key] = 0.
for item in opts:
print(item)
opt = item[0]
val = item[1]
if (opt == '-pcov1'):
prune_thresholds['cov1'] = val
if (opt == '-pcov2'):
prune_thresholds['cov2'] = val
if (opt == '-pfc1'):
prune_thresholds['fc1'] = val
if (opt == '-pfc2'):
prune_thresholds['fc2'] = val
if (opt == '-pfc3'):
prune_thresholds['fc3'] = val
if (opt == '-first_time'):
first_time_load = val
if (opt == '-file_name'):
file_name = val
if (opt == '-train'):
TRAIN = val
if (opt == '-prune'):
PRUNE = val
if (opt == '-parent_dir'):
parent_dir = val
if (opt == '-recover_rate'):
recover_rate = val
# print(recover_rate)
# sys.exit()
print('pruning thresholds are {}'.format(prune_thresholds))
except getopt.error, msg:
raise Usage(msg)
NUM_CLASSES = 10
dropout = 0.8
BATCH_SIZE = 128
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.001
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average.
DISPLAY_FREQ = 50
TEST = 1
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
DOCKER = 0
if (DOCKER == 1):
# base_model_name = '/root/data/20170206.pkl'
# model_name = '/root/data/pruning.pkl'
# mask_dir = '/root/data/mask.pkl'
base_model_name = '/root/20170206.pkl'
model_name = '/root/pruning'
mask_dir = '/root/mask'
else:
mask_dir = parent_dir + 'mask/'
weights_dir = parent_dir + 'weights/'
# model_name = 'test.pkl'
# model_name = '../tf_official_docker/tmp.pkl'
# if (TRAIN == True or PRUNE == True):
(weights_mask, biases_mask, soft_weight_mask,
soft_biase_mask) = initialize_weights_mask(
first_time_load, mask_dir, 'mask' + file_name + '.pkl')
if (TRAIN == True):
weights_mask, biases_mask = recover_weights(
weights_mask, biases_mask, soft_weight_mask, soft_biase_mask)
if (first_time_load):
weights_mask = {
'cov1': np.ones([5, 5, NUM_CHANNELS, 64]),
'cov2': np.ones([5, 5, 64, 64]),
'fc1': np.ones([6 * 6 * 64, 384]),
'fc2': np.ones([384, 192]),
'fc3': np.ones([192, NUM_CLASSES])
}
biases_mask = {
'cov1': np.ones([64]),
'cov2': np.ones([64]),
'fc1': np.ones([384]),
'fc2': np.ones([192]),
'fc3': np.ones([NUM_CLASSES])
}
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
if (TRAIN):
images_train, cls_train, labels_train = cifar10.load_training_data(
)
images_test, cls_test, labels_test = cifar10.load_test_data()
t_data = training_data(images_train, labels_train)
DATA_CNT = len(images_train)
NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE
else:
if (PRUNE):
images_test, cls_test, labels_test = cifar10.load_test_data()
else:
images_test, cls_test, labels_test = cifar10.load_test_data()
training_data_list = []
if (first_time_load):
PREV_MODEL_EXIST = 1
weights, biases = initialize_variables(
PREV_MODEL_EXIST, weights_dir + 'weights' + file_name + '.pkl')
else:
PREV_MODEL_EXIST = 1
weights, biases = initialize_variables(
PREV_MODEL_EXIST, weights_dir + 'weights' + file_name + '.pkl')
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, NUM_CLASSES])
keep_prob = tf.placeholder(tf.float32)
images = pre_process(x, TRAIN)
pred = cov_network(images, weights, biases, keep_prob)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=pred,
labels=y)
loss_value = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
global_step = tf.contrib.framework.get_or_create_global_step()
num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE
decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY)
# Decay the learning rate exponentially based on the number of steps.
# lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
# global_step,
# decay_steps,
# LEARNING_RATE_DECAY_FACTOR,
# staircase=True)
# opt = tf.train.GradientDescentOptimizer(lr)
opt = tf.train.AdamOptimizer(1e-5)
grads = opt.compute_gradients(loss_value)
org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads]
new_grads = mask_gradients(weights, org_grads, weights_mask, biases,
biases_mask)
# Apply gradients.
train_step = opt.apply_gradients(new_grads, global_step=global_step)
init = tf.global_variables_initializer()
# accuracy_list = np.zeros(50)
accuracy_list = np.zeros(20)
train_acc_list = []
# Launch the graph
print('Graph launching ..')
# config = tf.ConfigProto()
# config.gpu_options.per_process_gpu_memory_fraction = (0.7)
with tf.Session() as sess:
sess.run(init)
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
for key in keys:
sess.run(weights[key].assign(weights[key].eval() *
weights_mask[key]))
sess.run(biases[key].assign(biases[key].eval() *
biases_mask[key]))
print('pre train pruning info')
print('weights check')
prune_info(weights, 0)
print('mask check')
mask_info(weights_mask)
print(78 * '-')
start = time.time()
iter_cnt = 0
early_stoping = 0
if TRAIN == 1:
# for i in range(0,60000):
# for i in range(0,6):
for i in range(0, 30000):
iter_cnt = i
(batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE)
train_acc, cross_en = sess.run([accuracy, loss_value],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
if (i % DISPLAY_FREQ == 0):
print(
'This is the {}th iteration of {}pruning, time is {}'
.format(i, prune_thresholds,
time.time() - start))
print("accuracy is {} and cross entropy is {}".format(
train_acc, cross_en))
accuracy_list = np.concatenate(
(np.array([train_acc]), accuracy_list[0:19]))
# accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:49]))
# accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:4]))
if (i % (DISPLAY_FREQ * 50) == 0 and i != 0):
train_acc_list.append(train_acc)
save_pkl_model(weights, biases, weights_dir,
'weights' + file_name + '.pkl')
print("saved the network")
# if (np.mean(train_acc) > 0.5):
if (np.mean(accuracy_list) > 0.8 and train_acc > 0.82):
print(
"training accuracy is large, show the list: {}"
.format(accuracy_list))
# test_acc = sess.run(accuracy, feed_dict = {
# x: images_test,
# y: labels_test,
# keep_prob: 1.0})
accuracy_list = np.zeros(20)
# accuracy_list = np.zeros(50)
early_stoping = 1
# print('test accuracy is {}'.format(test_acc))
# if (test_acc > 0.78 and first_time_load):
# print('Exiting the training, test accuracy is {}'.format(test_acc))
break
_ = sess.run(train_step,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: dropout
})
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
# print("test accuracy in TRAINING is {}".format(test_acc))
if (TRAIN):
save_pkl_model(weights, biases, weights_dir,
'weights' + file_name + '.pkl')
with open('t_data/' + 'training_data' + file_name + '.pkl',
'wb') as f:
pickle.dump(train_acc_list, f)
if (PRUNE):
print('saving pruned model ...')
print(prune_thresholds)
f_name = compute_file_name(prune_thresholds)
prune_weights(prune_thresholds, weights, weights_mask, biases,
biases_mask, mask_dir, 'mask' + f_name + '.pkl',
recover_rate)
save_pkl_model(weights, biases, weights_dir,
'weights' + f_name + '.pkl')
return (test_acc, iter_cnt, early_stoping)
def main(argv=None):
if (argv is None):
argv = sys.argv
try:
try:
opts = argv
TRAIN = 1
NUMBER_OF_CLUSTER = 8
pretrain = 0
for item in opts:
print(item)
opt = item[0]
val = item[1]
if (opt == '-pcov'):
pruning_cov = val
if (opt == '-pfc'):
pruning_fc = val
if (opt == '-t'):
TRAIN = val
if (opt == '-cluster'):
NUMBER_OF_CLUSTER = val
if (opt == '-pretrain'):
pretrain = val
# print('pruning count is {}, {}'.format(pruning_cov, pruning_fc))
print('pretrain is {}'.format(pretrain))
except getopt.error, msg:
raise Usage(msg)
NUM_CLASSES = 10
dropout = 0.8 # probability of keep
BATCH_SIZE = 128
NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000
INITIAL_LEARNING_RATE = 0.001
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average.
DISPLAY_FREQ = 20
TEST = 0
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
DOCKER = 0
mask_dir = './weights_log/'
base_model_name = './data/20170206.pkl'
model_dir = './weights_log/'
# model_name = 'test.pkl'
# model_name = '../tf_official_docker/tmp.pkl'
PREV_MODEL_EXIST = 1
# cls_train returns as an integer, labels is the array
print('pruning on cov is {}. on fc is {}'.format(
pruning_cov, pruning_fc))
(weights_mask,
biases_mask) = initialize_weights_mask(0,
mask_dir + 'pruned_mask.pkl')
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
t_data = training_data(images_train, labels_train)
DATA_CNT = len(images_train)
NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE
training_data_list = []
weights_orgs, biases_orgs, biases, centroids_var, weights_index, cluster_index, centroids = initialize_variables(
PREV_MODEL_EXIST, NUMBER_OF_CLUSTER, pretrain)
weights = compute_weights(weights_index, centroids_var,
NUMBER_OF_CLUSTER)
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, NUM_CLASSES])
if (TRAIN == 1):
TRAIN_OR_TEST = 1
else:
TRAIN_OR_TEST = 0
keep_prob = tf.placeholder(tf.float32)
images = pre_process(x, TRAIN_OR_TEST)
# images = pre_process(x, 1)
pred = cov_network(images, weights, biases, keep_prob)
# print(pred)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(pred, y)
loss_value = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
global_step = tf.contrib.framework.get_or_create_global_step()
num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE
decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY)
# Decay the learning rate exponentially based on the number of steps.
lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE,
global_step,
decay_steps,
LEARNING_RATE_DECAY_FACTOR,
staircase=True)
opt = tf.train.GradientDescentOptimizer(lr)
grads = opt.compute_gradients(loss_value)
org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads]
new_grads = mask_gradients(weights, org_grads, weights_mask, biases,
biases_mask)
#
# Apply gradients.
train_step = opt.apply_gradients(new_grads, global_step=global_step)
# train_step = tf.train.GradientDescentOptimizer(INITIAL_LEARNING_RATE).minimize(loss_value)
# variable_averages = tf.train.ExponentialMovingAverage(
# MOVING_AVERAGE_DECAY, global_step)
# variables_averages_op = variable_averages.apply(tf.trainable_variables())
init = tf.global_variables_initializer()
accuracy_list = np.zeros(30)
accuracy_list = np.zeros(5)
# Launch the graph
print('Graph launching ..')
with tf.Session() as sess:
sess.run(init)
# restore model if exists
# if (os.path.isfile("tmp_20160130/model.meta")):
# op = tf.train.import_meta_graph("tmp_20160130/model.meta")
# op.restore(sess,tf.train.latest_checkpoint('tmp_20160130/'))
# print ("model found and restored")
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
for key in keys:
# sess.run(weights[key].assign(weights[key].eval()*weights_mask[key]))
sess.run(biases[key].assign(biases[key].eval() *
biases_mask[key]))
print('pre train pruning info')
prune_info(weights, 0)
print(78 * '-')
start = time.time()
if TRAIN == 1:
# for i in range(0,20):
for i in range(0, 60000):
(batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE)
train_acc, cross_en = sess.run([accuracy, loss_value],
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: 1.0
})
if (i % DISPLAY_FREQ == 0):
# prune_info(weights, 0)
print('This is the {}th iteration, time is {}'.format(
i,
time.time() - start))
print("accuracy is {} and cross entropy is {}".format(
train_acc, cross_en))
# accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:29]))
accuracy_list = np.concatenate(
(np.array([train_acc]), accuracy_list[0:4]))
if (np.mean(accuracy_list) > 0.8):
print(
"training accuracy is large, show the list: {}"
.format(accuracy_list))
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
# accuracy_list = np.zeros(30)
accuracy_list = np.zeros(5)
print('test accuracy is {}'.format(test_acc))
if (test_acc > 0.820):
print(
'Exiting the training, test accuracy is {}'
.format(test_acc))
break
_ = sess.run(train_step,
feed_dict={
x: batch_x,
y: batch_y,
keep_prob: dropout
})
if (TRAIN == 1):
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
weights_save = {}
centroids_save = {}
for key in keys:
centroids_save[key] = centroids_var[key].eval()
weights_save[key] = weights[key].eval()
with open('cluster_trained' + str(NUMBER_OF_CLUSTER) + '.pkl',
'wb') as f:
pickle.dump((weights_save, biases_orgs, cluster_index,
centroids_save), f)
test_acc = sess.run(accuracy,
feed_dict={
x: images_test,
y: labels_test,
keep_prob: 1.0
})
print("test accuracy is {}".format(test_acc))
# save_pkl_model(weights, biases, model_name)
return test_acc
# ['automobile',
# [ 'bird',
# [ 'cat',
# [ 'deer',
# [ 'dog',
# [ 'frog',
# [ 'horse',
# [ 'ship',
# [ 'truck']
#
# Load the training and testing sets
#
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
# display the size of the training and testing sets
#
print("Size of:")
print("- Training-set:\t\t{}".format(len(images_train)))
print("- Test-set:\t\t{}".format(len(images_test)))
# Output:
#
# Size of:
# Training-set: 50000
# Test-set: 10000
# crop the 32x32 CIFAR images down to 24x24:
#
def main(argv = None):
NUM_CLASSES = 10
BATCH_SIZE = 128
INITIAL_LEARNING_RATE = 0.001
LEARNING_RATE_DECAY_FACTOR = 0.1
NUM_EPOCHS_PER_DECAY = 350.0
MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average.
DISPLAY_FREQ = 20
TRAIN = 0
TEST = 1
TRAIN_OR_TEST = 0
NUM_CHANNELS = 3
DOCKER = 0
if (argv == None):
NUMBER_OF_CLUSTER = 8
else:
NUMBER_OF_CLUSTER = argv
mask_dir = './weights_log/'
# base_model_name = './data/20170206.pkl'
base_model_name = 'base.pkl'
parent_dir = './'
PREV_MODEL_EXIST = 1
(weights_mask,biases_mask)= initialize_weights_mask(0, parent_dir + 'masks/'+ base_model_name)
cifar10.maybe_download_and_extract()
class_names = cifar10.load_class_names()
images_train, cls_train, labels_train = cifar10.load_training_data()
images_test, cls_test, labels_test = cifar10.load_test_data()
test_data = format_data(images_test, labels_test)
DATA_CNT = len(images_train)
NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE
weights, biases = initialize_variables(PREV_MODEL_EXIST,
parent_dir + 'weights/' + base_model_name)
x = tf.placeholder(tf.float32, [None, 32, 32, 3])
y = tf.placeholder(tf.float32, [None, NUM_CLASSES])
keep_prob = tf.placeholder(tf.float32)
images = pre_process(x, 0)
pred = cov_network(images, weights, biases, keep_prob)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y)
loss_value = tf.reduce_mean(cross_entropy)
correct_prediction = tf.equal(tf.argmax(pred,1), tf.argmax(y,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
global_step = tf.contrib.framework.get_or_create_global_step()
init = tf.global_variables_initializer()
print('Graph launching ..')
with tf.Session() as sess:
sess.run(init)
# prune the network
keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3']
for key in keys:
sess.run(weights[key].assign(weights[key].eval()*weights_mask[key]))
sess.run(biases[key].assign(biases[key].eval()*biases_mask[key]))
print('quantisation process starts..')
prune_info(weights, 0)
NUMBER_OF_BATCH = 10000 / BATCH_SIZE
t_acc = []
test_data = format_data(images_test, labels_test)
for i in range(0,NUMBER_OF_BATCH):
(batch_x, batch_y) = test_data.feed_next_batch(BATCH_SIZE)
test_acc = sess.run(accuracy, feed_dict = {
x: batch_x,
y: batch_y,
keep_prob: 1.0})
t_acc.append(test_acc)
print("test accuracy is {}".format(t_acc))
test_acc = np.mean(t_acc)
print('Pre quantisation model has an accuracy of {}'.format(test_acc))
print(78*'-')
start = time.time()
keys = ['cov1','cov2','fc1','fc2','fc3']
weights_val = {}
centroids = {}
cluster_index = {}
weights_orgs = {}
biases_orgs = {}
for key in keys:
weight_org = weights[key].eval()
weight= weight_org.flatten()
weight_val = weight[weight != 0]
data = np.expand_dims(weight_val, axis = 1)
print(np.shape(data))
# use kmeans to cluster
# kmeans = KMeans(n_clusters= NUMBER_OF_CLUSTER, random_state=1).fit(data)
kmeans = MiniBatchKMeans(n_clusters= NUMBER_OF_CLUSTER, random_state=0, init='k-means++').fit(data)
centroid = kmeans.cluster_centers_
# add centroid value
centroids[key] = centroid
# add index value
# indexs are stored in weight_org
index = kmeans.labels_ + 1
for w in np.nditer(weight_org, op_flags=['readwrite']):
if (w != 0):
w[...] = kmeans.predict(w)+1
# sys.exit()
cluster_index[key] = weight_org
weights_orgs[key] = weights[key].eval()
biases_orgs[key] = biases[key].eval()
print('quantisation done')
print(78*'-')
print("test accuracy is {}".format(test_acc))
with open(parent_dir + 'weights/' + 'weightspt' +str(NUMBER_OF_CLUSTER)+'.pkl','wb') as f:
pickle.dump((weights_orgs, biases_orgs, cluster_index,centroids),f)