def load_cifar10(): # make directory if not exist if not os.path.isdir("data"): os.mkdir("data") if not os.path.isdir("data/CIFAR-10"): os.mkdir("data/CIFAR-10") # download and extract if not done yet # data is downloaded from data_url = "https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz" # to data_path = "data/CIFAR-10/" cifar10.data_path = "data/CIFAR-10/" cifar10.maybe_download_and_extract() # load data x_train, y_train_cls, y_train = cifar10.load_training_data() x_test, y_test_cls, y_test = cifar10.load_test_data() class_names = cifar10.load_class_names() x_train = x_train.astype(np.float32) y_train_cls = y_train_cls.astype(np.int32) y_train = y_train.astype(np.float32) x_test = x_test.astype(np.float32) y_test_cls = y_test_cls.astype(np.int32) y_test = y_test.astype(np.float32) data = (x_train, y_train_cls, y_train, x_test, y_test_cls, y_test, class_names) return data
def main(): class_names = cifar10.load_class_names() images_train, cls_idx_train, labels_train = cifar10.load_training_data() images_test, cls_idx_test, labels_test = cifar10.load_test_data() #Plot the first 9 training images and labels plot_9images(images=images_train[0:9], cls_idx_true=cls_idx_train[0:9], all_cls_names=class_names, smooth=True) # Build your predictor w1, b1, w2, b2 = train(images_train, labels_train, images_test, cls_idx_test) # Visualize your prediction print('--------------------------------Neural Network--------------------------------') samples = random.sample(range(len(images_test)), 9) plot_9images(images=images_test[samples], cls_idx_true=cls_idx_test[samples], cls_idx_pred=predict(images_test[samples], w1, b1, w2, b2), all_cls_names=class_names, smooth=True) print(f'\nAccuracy: {(predict(images_test, w1, b1, w2, b2) == cls_idx_test).mean() * 100}%\n') knn_idx = kNN(images_train, cls_idx_train, images_test) print('-------------------------------k-Nearest Neighbor-------------------------------') samples = random.sample(range(len(images_test)), 9) plot_9images(images=images_test[samples], cls_idx_true=cls_idx_test[samples], cls_idx_pred=knn_idx[samples], all_cls_names=class_names, smooth=True) print(f'\nAccuracy: {(knn_idx == cls_idx_test).mean() * 100}%\n')
def download_dataset(): cifar10.maybe_download_and_extract() class_names = cifar10.load_class_names() images_train, cls_train, labels_train = cifar10.load_training_data() images_test, cls_test, labels_test = cifar10.load_test_data() print("Size of:") print("- Training-set:\t\t{}".format(len(images_train))) print("- Test-set:\t\t{}".format(len(images_test))) return class_names, images_train, cls_train, labels_train, images_test, cls_test, labels_test
def train(): class_names = cifar10.load_class_names() print(class_names) images_train, cls_train, labels_train = cifar10.load_training_data() images_test, cls_test, labels_test = cifar10.load_test_data() with open(r'E:\tmp\CIFAR-10\inception_cifar10_test.pkl', mode='rb') as file: transfer_values_test = pickle.load(file) with open(r'E:\tmp\CIFAR-10\inception_cifar10_train.pkl', mode='rb') as file: transfer_values_train = pickle.load(file) print(transfer_values_train.shape) print(type(transfer_values_train)) model=tl_Inception(r'E:\tmp\tl_inception') model.fit(transfer_values_train,labels_train,transfer_values_test,labels_test)
def main(): class_names = cifar10.load_class_names() images_train, cls_idx_train, labels_train = cifar10.load_training_data() images_test, cls_idx_test, labels_test = cifar10.load_test_data() # Plot the first 9 training images and labels plot_9images(images=images_train[0:9], cls_idx_true=cls_idx_train[0:9], all_cls_names=class_names, smooth=True) # Build your predictor train(images_train, labels_train) # Visualize your prediction samples = random.sample(range(len(images_test)), 9) plot_9images(images=images_test[samples], cls_idx_true=cls_idx_test[samples], cls_idx_pred=predict(images_test[samples]), all_cls_names=class_names, smooth=True)
def test(): class_names = cifar10.load_class_names() test_batch_size = 64 # images_train, cls_train, labels_train = cifar10.load_training_data() images_test, cls_test, labels_test = cifar10.load_test_data() with open(r'E:\tmp\CIFAR-10\inception_cifar10_test.pkl', mode='rb') as file: transfer_values_test = pickle.load(file) # with open(r'E:\tmp\CIFAR-10\inception_cifar10_train.pkl', mode='rb') as file: # transfer_values_train = pickle.load(file) model = tl_Inception(r'E:\tmp\tl_inception') score_list = [] _X = transfer_values_test _Y = labels_test _im = images_test for i in range(1): _x,_y = random_batch(_X,_Y,test_batch_size) pred = model.predict(_x) print(pred) true_cls = np.argmax(_y,axis=1) print(true_cls) print('score is ', 1-np.mean(pred != true_cls)) print('wrong classified samples ',np.sum(pred != true_cls)) score_list.append(1-np.mean(pred != true_cls)) print('mean score is ',np.mean(score_list)) #test with plot im_list = np.random.choice(10000,size=10,replace=False) im = _im[im_list] label = np.argmax(_Y[im_list],axis=1) _x = _X[im_list] pred = model.predict(_x) for i in range(10): print(im[i].shape) plot_helper(im[i]) plot_helper(_x[i]) print(label[i],'-',pred[i]) print(label[i],'',class_names[label[i]],'-',pred[i],class_names[pred[i]])
import time from datetime import timedelta import math import os %matplotlib inline cifar10.data_path = "C:\\ML_Data\\_CIFAR-10\\" # download the CIFAR-10 image data if it's not already local # cifar10.maybe_download_and_extract() # Class names # class_names = cifar10.load_class_names() class_names # Output: # # ['airplane', # ['automobile', # [ 'bird', # [ 'cat', # [ 'deer', # [ 'dog', # [ 'frog', # [ 'horse', # [ 'ship', # [ 'truck'] #
import tensorflow as tf import matplotlib.pyplot as plt import numpy as np import time from datetime import timedelta import cifar10 cifar10.download() print(cifar10.load_class_names()) train_img, train_cls, train_labels = cifar10.load_training_data() test_img, test_cls, test_labels = cifar10.load_test_data() print('Training set:', len(train_img), 'Testing set:', len(test_img)) x = tf.placeholder(tf.float32, [None, 32, 32, 3]) y_true = tf.placeholder(tf.float32, [None, 10]) def conv_layer(input, size_in, size_out, use_pooling=True): w = tf.Variable(tf.truncated_normal([3, 3, size_in, size_out], stddev=0.1)) b = tf.Variable(tf.constant(0.1, shape=[size_out])) conv = tf.nn.conv2d(input, w, strides=[1, 1, 1, 1], padding='SAME') y = tf.nn.relu(conv + b) if use_pooling: y = tf.nn.max_pool(y, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='SAME')
def main(argv=None): if (argv is None): argv = sys.argv try: try: opts = argv first_time_load = False parent_dir = '' keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] prune_thresholds = {} TRAIN = True for key in keys: prune_thresholds[key] = 0. for item in opts: print(item) opt = item[0] val = item[1] if (opt == '-pcov1'): prune_thresholds['cov1'] = val if (opt == '-pcov2'): prune_thresholds['cov2'] = val if (opt == '-pfc1'): prune_thresholds['fc1'] = val if (opt == '-pfc2'): prune_thresholds['fc2'] = val if (opt == '-pfc3'): prune_thresholds['fc3'] = val if (opt == '-first_time'): first_time_load = val if (opt == '-file_name'): file_name = val if (opt == '-train'): TRAIN = val if (opt == '-prune'): PRUNE = val if (opt == '-parent_dir'): parent_dir = val if (opt == '-recover_rate'): recover_rate = val # print(recover_rate) # sys.exit() print('pruning thresholds are {}'.format(prune_thresholds)) except getopt.error, msg: raise Usage(msg) NUM_CLASSES = 10 dropout = 0.8 BATCH_SIZE = 128 NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000 INITIAL_LEARNING_RATE = 0.001 LEARNING_RATE_DECAY_FACTOR = 0.1 NUM_EPOCHS_PER_DECAY = 350.0 MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average. DISPLAY_FREQ = 50 TEST = 1 TRAIN_OR_TEST = 0 NUM_CHANNELS = 3 DOCKER = 0 if (DOCKER == 1): # base_model_name = '/root/data/20170206.pkl' # model_name = '/root/data/pruning.pkl' # mask_dir = '/root/data/mask.pkl' base_model_name = '/root/20170206.pkl' model_name = '/root/pruning' mask_dir = '/root/mask' else: mask_dir = parent_dir + 'mask/' weights_dir = parent_dir + 'weights/' # model_name = 'test.pkl' # model_name = '../tf_official_docker/tmp.pkl' # if (TRAIN == True or PRUNE == True): (weights_mask, biases_mask, soft_weight_mask, soft_biase_mask) = initialize_weights_mask( first_time_load, mask_dir, 'mask' + file_name + '.pkl') if (TRAIN == True): weights_mask, biases_mask = recover_weights( weights_mask, biases_mask, soft_weight_mask, soft_biase_mask) if (first_time_load): weights_mask = { 'cov1': np.ones([5, 5, NUM_CHANNELS, 64]), 'cov2': np.ones([5, 5, 64, 64]), 'fc1': np.ones([6 * 6 * 64, 384]), 'fc2': np.ones([384, 192]), 'fc3': np.ones([192, NUM_CLASSES]) } biases_mask = { 'cov1': np.ones([64]), 'cov2': np.ones([64]), 'fc1': np.ones([384]), 'fc2': np.ones([192]), 'fc3': np.ones([NUM_CLASSES]) } cifar10.maybe_download_and_extract() class_names = cifar10.load_class_names() if (TRAIN): images_train, cls_train, labels_train = cifar10.load_training_data( ) images_test, cls_test, labels_test = cifar10.load_test_data() t_data = training_data(images_train, labels_train) DATA_CNT = len(images_train) NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE else: if (PRUNE): images_test, cls_test, labels_test = cifar10.load_test_data() else: images_test, cls_test, labels_test = cifar10.load_test_data() training_data_list = [] if (first_time_load): PREV_MODEL_EXIST = 1 weights, biases = initialize_variables( PREV_MODEL_EXIST, weights_dir + 'weights' + file_name + '.pkl') else: PREV_MODEL_EXIST = 1 weights, biases = initialize_variables( PREV_MODEL_EXIST, weights_dir + 'weights' + file_name + '.pkl') x = tf.placeholder(tf.float32, [None, 32, 32, 3]) y = tf.placeholder(tf.float32, [None, NUM_CLASSES]) keep_prob = tf.placeholder(tf.float32) images = pre_process(x, TRAIN) pred = cov_network(images, weights, biases, keep_prob) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y) loss_value = tf.reduce_mean(cross_entropy) correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) saver = tf.train.Saver() global_step = tf.contrib.framework.get_or_create_global_step() num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY) # Decay the learning rate exponentially based on the number of steps. # lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE, # global_step, # decay_steps, # LEARNING_RATE_DECAY_FACTOR, # staircase=True) # opt = tf.train.GradientDescentOptimizer(lr) opt = tf.train.AdamOptimizer(1e-5) grads = opt.compute_gradients(loss_value) org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads] new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask) # Apply gradients. train_step = opt.apply_gradients(new_grads, global_step=global_step) init = tf.global_variables_initializer() # accuracy_list = np.zeros(50) accuracy_list = np.zeros(20) train_acc_list = [] # Launch the graph print('Graph launching ..') # config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = (0.7) with tf.Session() as sess: sess.run(init) keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] for key in keys: sess.run(weights[key].assign(weights[key].eval() * weights_mask[key])) sess.run(biases[key].assign(biases[key].eval() * biases_mask[key])) print('pre train pruning info') print('weights check') prune_info(weights, 0) print('mask check') mask_info(weights_mask) print(78 * '-') start = time.time() iter_cnt = 0 early_stoping = 0 if TRAIN == 1: # for i in range(0,60000): # for i in range(0,6): for i in range(0, 30000): iter_cnt = i (batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE) train_acc, cross_en = sess.run([accuracy, loss_value], feed_dict={ x: batch_x, y: batch_y, keep_prob: 1.0 }) if (i % DISPLAY_FREQ == 0): print( 'This is the {}th iteration of {}pruning, time is {}' .format(i, prune_thresholds, time.time() - start)) print("accuracy is {} and cross entropy is {}".format( train_acc, cross_en)) accuracy_list = np.concatenate( (np.array([train_acc]), accuracy_list[0:19])) # accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:49])) # accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:4])) if (i % (DISPLAY_FREQ * 50) == 0 and i != 0): train_acc_list.append(train_acc) save_pkl_model(weights, biases, weights_dir, 'weights' + file_name + '.pkl') print("saved the network") # if (np.mean(train_acc) > 0.5): if (np.mean(accuracy_list) > 0.8 and train_acc > 0.82): print( "training accuracy is large, show the list: {}" .format(accuracy_list)) # test_acc = sess.run(accuracy, feed_dict = { # x: images_test, # y: labels_test, # keep_prob: 1.0}) accuracy_list = np.zeros(20) # accuracy_list = np.zeros(50) early_stoping = 1 # print('test accuracy is {}'.format(test_acc)) # if (test_acc > 0.78 and first_time_load): # print('Exiting the training, test accuracy is {}'.format(test_acc)) break _ = sess.run(train_step, feed_dict={ x: batch_x, y: batch_y, keep_prob: dropout }) test_acc = sess.run(accuracy, feed_dict={ x: images_test, y: labels_test, keep_prob: 1.0 }) # print("test accuracy in TRAINING is {}".format(test_acc)) if (TRAIN): save_pkl_model(weights, biases, weights_dir, 'weights' + file_name + '.pkl') with open('t_data/' + 'training_data' + file_name + '.pkl', 'wb') as f: pickle.dump(train_acc_list, f) if (PRUNE): print('saving pruned model ...') print(prune_thresholds) f_name = compute_file_name(prune_thresholds) prune_weights(prune_thresholds, weights, weights_mask, biases, biases_mask, mask_dir, 'mask' + f_name + '.pkl', recover_rate) save_pkl_model(weights, biases, weights_dir, 'weights' + f_name + '.pkl') return (test_acc, iter_cnt, early_stoping)
def main(argv = None): if (argv is None): argv = sys.argv try: try: opts = argv TRAIN = 1 pretrain = 0 for item in opts: print (item) opt = item[0] val = item[1] if (opt == '-t'): TRAIN = val if (opt == '-q_bits'): q_bits = val if (opt == '-parent_dir'): parent_dir = val if (opt == '-base_model'): base_model = val if (opt == '-pretrain'): pretrain = val print('pretrain is {}'.format(pretrain)) except getopt.error, msg: raise Usage(msg) NUM_CLASSES = 10 dropout = 0.8 # probability of keep BATCH_SIZE = 128 NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000 INITIAL_LEARNING_RATE = 0.001 LEARNING_RATE_DECAY_FACTOR = 0.1 NUM_EPOCHS_PER_DECAY = 350.0 MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average. DISPLAY_FREQ = 20 TEST = 0 TRAIN_OR_TEST = 0 NUM_CHANNELS = 3 DOCKER = 0 mask_dir = parent_dir model_dir = parent_dir PREV_MODEL_EXIST = 1 (weights_mask,biases_mask)= initialize_weights_mask(0, mask_dir + 'masks/' + 'base.pkl' ) cifar10.maybe_download_and_extract() class_names = cifar10.load_class_names() images_train, cls_train, labels_train = cifar10.load_training_data() images_test, cls_test, labels_test = cifar10.load_test_data() t_data = format_data(images_train, labels_train) test_data = format_data(images_test, labels_test) DATA_CNT = len(images_train) NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE training_data_list = [] weights, biases = initialize_variables(PREV_MODEL_EXIST, parent_dir, q_bits, pretrain) weights, biases = compute_weights_nbits(weights, biases, q_bits) x = tf.placeholder(tf.float32, [None, 32, 32, 3]) y = tf.placeholder(tf.float32, [None, NUM_CLASSES]) if (TRAIN == 1): TRAIN_OR_TEST = 1 else: TRAIN_OR_TEST = 0 keep_prob = tf.placeholder(tf.float32) images = pre_process(x, TRAIN_OR_TEST) # images = pre_process(x, 1) pred = cov_network(images, weights, biases, keep_prob) # print(pred) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y) loss_value = tf.reduce_mean(cross_entropy) correct_prediction = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) saver = tf.train.Saver() global_step = tf.contrib.framework.get_or_create_global_step() num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY) # Decay the learning rate exponentially based on the number of steps. lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE, global_step, decay_steps, LEARNING_RATE_DECAY_FACTOR, staircase=True) opt = tf.train.GradientDescentOptimizer(lr) grads = opt.compute_gradients(loss_value) org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads] new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask) # Apply gradients. train_step = opt.apply_gradients(new_grads, global_step=global_step) init = tf.global_variables_initializer() accuracy_list = np.zeros(30) accuracy_list = np.zeros(5) # Launch the graph print('Graph launching ..') with tf.Session() as sess: sess.run(init) keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] for key in keys: sess.run(weights[key].assign(weights[key].eval()*weights_mask[key])) # sess.run(biases[key].assign(biases[key].eval()*biases_mask[key])) print('pre train pruning info') prune_info(weights, 0) print(78*'-') start = time.time() if TRAIN == 1: for i in range(0,10000): (batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE) train_acc, cross_en = sess.run([accuracy, loss_value], feed_dict = { x: batch_x, y: batch_y, keep_prob: 1.0}) if (i % DISPLAY_FREQ == 0): # prune_info(weights, 0) print('This is the {}th iteration, time is {}'.format( i, time.time() - start )) print("accuracy is {} and cross entropy is {}".format( train_acc, cross_en )) # accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:29])) accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:4])) if (np.mean(accuracy_list) > 0.8): print("training accuracy is large, show the list: {}".format(accuracy_list)) NUMBER_OF_BATCH = 10000 / BATCH_SIZE t_acc = [] for i in range(0,NUMBER_OF_BATCH): (batch_x, batch_y) = test_data.feed_next_batch(BATCH_SIZE) test_acc = sess.run(accuracy, feed_dict = { x: batch_x, y: batch_y, keep_prob: 1.0}) t_acc.append(test_acc) print("test accuracy is {}".format(t_acc)) test_acc = np.mean(t_acc) accuracy_list = np.zeros(5) print('test accuracy is {}'.format(test_acc)) if (test_acc > 0.823): print('Exiting the training, test accuracy is {}'.format(test_acc)) break _ = sess.run(train_step, feed_dict = { x: batch_x, y: batch_y, keep_prob: dropout}) if (TRAIN == 1): keys = ['cov1','cov2','fc1','fc2','fc3'] weights_save = {} centroids_save = {} for key in keys: centroids_save[key] = centroids_var[key].eval() weights_save[key] = weights[key].eval() with open(parent_dir + 'weights/'+ 'weights'+str(q_bits)+'.pkl','wb') as f: pickle.dump((weights_save, biases_orgs, cluster_index,centroids_save),f) NUMBER_OF_BATCH = 10000 / BATCH_SIZE t_acc = [] for i in range(0,NUMBER_OF_BATCH): (batch_x, batch_y) = test_data.feed_next_batch(BATCH_SIZE) test_acc = sess.run(accuracy, feed_dict = { x: batch_x, y: batch_y, keep_prob: 1.0}) t_acc.append(test_acc) print("test accuracy is {}".format(t_acc)) # save_pkl_model(weights, biases, model_name) return np.mean(t_acc)
def main(argv=None): if (argv is None): argv = sys.argv try: try: opts = argv first_time_load = True parent_dir = './' keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] prune_thresholds = {} WITH_BIASES = False next_iter_save = False for key in keys: prune_thresholds[key] = 0. for item in opts: print(item) opt = item[0] val = item[1] if (opt == '-pcov1'): prune_thresholds['cov1'] = val if (opt == '-pcov2'): prune_thresholds['cov2'] = val if (opt == '-pfc1'): prune_thresholds['fc1'] = val if (opt == '-pfc2'): prune_thresholds['fc2'] = val if (opt == '-pfc3'): prune_thresholds['fc3'] = val if (opt == '-first_time'): first_time_load = val if (opt == '-init_file_name'): init_file_name = val if (opt == '-train'): TRAIN = val if (opt == '-prune'): PRUNE = val if (opt == '-parent_dir'): parent_dir = val if (opt == '-lr'): lr = val if (opt == '-with_biases'): WITH_BIASES = val if (opt == '-cRates'): cRates = val if (opt == '-iter_cnt'): iter_cnt = val if (opt == '-save'): next_iter_save = val if (opt == '-org_file_name'): org_file_name = val print('pruning thresholds are {}'.format(prune_thresholds)) except getopt.error, msg: raise Usage(msg) NUM_CLASSES = 10 dropout = 0.5 BATCH_SIZE = 128 NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000 INITIAL_LEARNING_RATE = 0.001 INITIAL_LEARNING_RATE = lr LEARNING_RATE_DECAY_FACTOR = 0.1 NUM_EPOCHS_PER_DECAY = 350.0 MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average. DISPLAY_FREQ = 50 TEST = 1 TRAIN_OR_TEST = 0 NUM_CHANNELS = 3 DOCKER = 0 if (DOCKER == 1): base_model_name = '/root/20170206.pkl' model_name = '/root/pruning' mask_dir = '/root/mask' else: mask_dir = parent_dir weights_dir = parent_dir # model_name = 'test.pkl' # model_name = '../tf_official_docker/tmp.pkl' if (next_iter_save): file_name_part = org_file_name else: file_name_part = compute_file_name(cRates) (weights_mask, biases_mask) = initialize_weights_mask( first_time_load, mask_dir, 'mask_crate' + file_name_part + '.pkl') cifar10.maybe_download_and_extract() class_names = cifar10.load_class_names() if (TRAIN): images_train, cls_train, labels_train = cifar10.load_training_data( ) images_test, cls_test, labels_test = cifar10.load_test_data() t_data = training_data(images_train, labels_train) DATA_CNT = len(images_train) NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE else: if (PRUNE): images_test, cls_test, labels_test = cifar10.load_test_data() else: images_test, cls_test, labels_test = cifar10.load_test_data() training_data_list = [] if (first_time_load): PREV_MODEL_EXIST = 0 weights, biases = initialize_variables(PREV_MODEL_EXIST, '') else: PREV_MODEL_EXIST = 1 weights_dir = parent_dir if (next_iter_save): file_name_part = org_file_name else: file_name_part = compute_file_name(cRates) weights, biases = initialize_variables( PREV_MODEL_EXIST, parent_dir + 'weight_crate' + file_name_part + '.pkl') x = tf.placeholder(tf.float32, [None, 32, 32, 3]) y = tf.placeholder(tf.float32, [None, NUM_CLASSES]) keep_prob = tf.placeholder(tf.float32) images = pre_process(x, TRAIN) no_preprocesss_images = pre_process(x, False) weights_new = {} keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] for key in keys: weights_new[key] = weights[key] * tf.constant(weights_mask[key], dtype=tf.float32) pred = cov_network(images, weights_new, biases, keep_prob) test_pred = cov_network(no_preprocesss_images, weights_new, biases, keep_prob) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits=pred, labels=y) loss_value = tf.reduce_mean(cross_entropy) correct_prediction = tf.equal(tf.argmax(test_pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) opt = tf.train.AdamOptimizer(lr) grads = opt.compute_gradients(loss_value) org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads] # new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask) # Apply gradients. train_step = opt.apply_gradients(org_grads) init = tf.global_variables_initializer() accuracy_list = np.zeros(20) # accuracy_list = np.zeros(5) train_acc_list = [] # Launch the graph print('Graph launching ..') # config = tf.ConfigProto() # config.gpu_options.per_process_gpu_memory_fraction = (0.7) # gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.333) with tf.Session() as sess: sess.run(init) print('pre train pruning info') prune_info(weights_new, 0) print(78 * '-') start = time.time() if TRAIN == 1: # for i in range(0,60000): # for i in range(0,6): for i in range(0, 20000): (batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE) train_acc, cross_en = sess.run([accuracy, loss_value], feed_dict={ x: batch_x, y: batch_y, keep_prob: 1.0 }) if (i % DISPLAY_FREQ == 0): print( 'This is the {}th iteration of {}pruning, time is {}' .format(i, cRates, time.time() - start)) print("accuracy is {} and cross entropy is {}".format( train_acc, cross_en)) # accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:49])) accuracy_list = np.concatenate( (np.array([train_acc]), accuracy_list[0:19])) # accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:4])) if (i % (DISPLAY_FREQ * 50) == 0 and i != 0): train_acc_list.append(train_acc) file_name_part = compute_file_name(cRates) file_name = 'weight_crate' + str( file_name_part) + '.pkl' save_pkl_model(weights, biases, parent_dir, file_name) print("saved the network") if (np.mean(accuracy_list) > 0.81 and train_acc >= 0.82): prune_info(weights_new, 0) print(accuracy_list) accuracy_list = np.zeros(20) test_acc = sess.run(accuracy, feed_dict={ x: images_test, y: labels_test, keep_prob: 1.0 }) print(test_acc) if (test_acc > 0.823): print( "training accuracy is large, show the list: {}" .format(accuracy_list)) break _ = sess.run(train_step, feed_dict={ x: batch_x, y: batch_y, keep_prob: dropout }) test_acc = sess.run(accuracy, feed_dict={ x: images_test, y: labels_test, keep_prob: 1.0 }) print("test accuracy is {}".format(test_acc)) if (next_iter_save): print('saving for the next iteration of dynamic surgery') file_name_part = compute_file_name(cRates) file_name = 'weight_crate' + file_name_part + '.pkl' save_pkl_model(weights, biases, parent_dir, file_name) file_name_part = compute_file_name(cRates) with open(parent_dir + 'mask_crate' + file_name_part + '.pkl', 'wb') as f: pickle.dump(weights_mask, f) if (TRAIN): file_name_part = compute_file_name(cRates) file_name = 'weight_crate' + file_name_part + '.pkl' save_pkl_model(weights, biases, parent_dir, file_name) with open(parent_dir + 'training_data' + file_name + '.pkl', 'wb') as f: pickle.dump(train_acc_list, f) if (PRUNE): print('saving pruned model ...') file_name_part = compute_file_name(cRates) prune_weights(prune_thresholds, weights, biases, weights_mask, cRates, iter_cnt, parent_dir) file_name = 'weight_crate' + file_name_part + '.pkl' print('saving pruned network') save_pkl_model(weights, biases, parent_dir, file_name) return test_acc
def main(argv=None): if (argv is None): argv = sys.argv try: try: opts = argv TRAIN = 1 NUMBER_OF_CLUSTER = 8 pretrain = 0 for item in opts: print(item) opt = item[0] val = item[1] if (opt == '-pcov'): pruning_cov = val if (opt == '-pfc'): pruning_fc = val if (opt == '-t'): TRAIN = val if (opt == '-cluster'): NUMBER_OF_CLUSTER = val if (opt == '-pretrain'): pretrain = val # print('pruning count is {}, {}'.format(pruning_cov, pruning_fc)) print('pretrain is {}'.format(pretrain)) except getopt.error, msg: raise Usage(msg) NUM_CLASSES = 10 dropout = 0.8 # probability of keep BATCH_SIZE = 128 NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000 INITIAL_LEARNING_RATE = 0.001 LEARNING_RATE_DECAY_FACTOR = 0.1 NUM_EPOCHS_PER_DECAY = 350.0 MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average. DISPLAY_FREQ = 20 TEST = 0 TRAIN_OR_TEST = 0 NUM_CHANNELS = 3 DOCKER = 0 mask_dir = './weights_log/' base_model_name = './data/20170206.pkl' model_dir = './weights_log/' # model_name = 'test.pkl' # model_name = '../tf_official_docker/tmp.pkl' PREV_MODEL_EXIST = 1 # cls_train returns as an integer, labels is the array print('pruning on cov is {}. on fc is {}'.format( pruning_cov, pruning_fc)) (weights_mask, biases_mask) = initialize_weights_mask(0, mask_dir + 'pruned_mask.pkl') cifar10.maybe_download_and_extract() class_names = cifar10.load_class_names() images_train, cls_train, labels_train = cifar10.load_training_data() images_test, cls_test, labels_test = cifar10.load_test_data() t_data = training_data(images_train, labels_train) DATA_CNT = len(images_train) NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE training_data_list = [] weights_orgs, biases_orgs, biases, centroids_var, weights_index, cluster_index, centroids = initialize_variables( PREV_MODEL_EXIST, NUMBER_OF_CLUSTER, pretrain) weights = compute_weights(weights_index, centroids_var, NUMBER_OF_CLUSTER) x = tf.placeholder(tf.float32, [None, 32, 32, 3]) y = tf.placeholder(tf.float32, [None, NUM_CLASSES]) if (TRAIN == 1): TRAIN_OR_TEST = 1 else: TRAIN_OR_TEST = 0 keep_prob = tf.placeholder(tf.float32) images = pre_process(x, TRAIN_OR_TEST) # images = pre_process(x, 1) pred = cov_network(images, weights, biases, keep_prob) # print(pred) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(pred, y) loss_value = tf.reduce_mean(cross_entropy) correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) saver = tf.train.Saver() global_step = tf.contrib.framework.get_or_create_global_step() num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY) # Decay the learning rate exponentially based on the number of steps. lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE, global_step, decay_steps, LEARNING_RATE_DECAY_FACTOR, staircase=True) opt = tf.train.GradientDescentOptimizer(lr) grads = opt.compute_gradients(loss_value) org_grads = [(ClipIfNotNone(grad), var) for grad, var in grads] new_grads = mask_gradients(weights, org_grads, weights_mask, biases, biases_mask) # # Apply gradients. train_step = opt.apply_gradients(new_grads, global_step=global_step) # train_step = tf.train.GradientDescentOptimizer(INITIAL_LEARNING_RATE).minimize(loss_value) # variable_averages = tf.train.ExponentialMovingAverage( # MOVING_AVERAGE_DECAY, global_step) # variables_averages_op = variable_averages.apply(tf.trainable_variables()) init = tf.global_variables_initializer() accuracy_list = np.zeros(30) accuracy_list = np.zeros(5) # Launch the graph print('Graph launching ..') with tf.Session() as sess: sess.run(init) # restore model if exists # if (os.path.isfile("tmp_20160130/model.meta")): # op = tf.train.import_meta_graph("tmp_20160130/model.meta") # op.restore(sess,tf.train.latest_checkpoint('tmp_20160130/')) # print ("model found and restored") keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] for key in keys: # sess.run(weights[key].assign(weights[key].eval()*weights_mask[key])) sess.run(biases[key].assign(biases[key].eval() * biases_mask[key])) print('pre train pruning info') prune_info(weights, 0) print(78 * '-') start = time.time() if TRAIN == 1: # for i in range(0,20): for i in range(0, 60000): (batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE) train_acc, cross_en = sess.run([accuracy, loss_value], feed_dict={ x: batch_x, y: batch_y, keep_prob: 1.0 }) if (i % DISPLAY_FREQ == 0): # prune_info(weights, 0) print('This is the {}th iteration, time is {}'.format( i, time.time() - start)) print("accuracy is {} and cross entropy is {}".format( train_acc, cross_en)) # accuracy_list = np.concatenate((np.array([train_acc]),accuracy_list[0:29])) accuracy_list = np.concatenate( (np.array([train_acc]), accuracy_list[0:4])) if (np.mean(accuracy_list) > 0.8): print( "training accuracy is large, show the list: {}" .format(accuracy_list)) test_acc = sess.run(accuracy, feed_dict={ x: images_test, y: labels_test, keep_prob: 1.0 }) # accuracy_list = np.zeros(30) accuracy_list = np.zeros(5) print('test accuracy is {}'.format(test_acc)) if (test_acc > 0.820): print( 'Exiting the training, test accuracy is {}' .format(test_acc)) break _ = sess.run(train_step, feed_dict={ x: batch_x, y: batch_y, keep_prob: dropout }) if (TRAIN == 1): keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] weights_save = {} centroids_save = {} for key in keys: centroids_save[key] = centroids_var[key].eval() weights_save[key] = weights[key].eval() with open('cluster_trained' + str(NUMBER_OF_CLUSTER) + '.pkl', 'wb') as f: pickle.dump((weights_save, biases_orgs, cluster_index, centroids_save), f) test_acc = sess.run(accuracy, feed_dict={ x: images_test, y: labels_test, keep_prob: 1.0 }) print("test accuracy is {}".format(test_acc)) # save_pkl_model(weights, biases, model_name) return test_acc
for i in [15,42,81,156,501,1002,4500,6750,9000,11250,13500,15750]: for j in ["TrialValidation1","TrialValidation2","TrialValidation3"]: #xxx = "Don't run" xxx = "Run" num_iterations = 1000000 train_batch_size = 64 imgchosen = 2473 #should be SCr Numberoftrainingsamples = i Numberoftestsamples = 5250 require_improvement = 1500 _images_per_file = int(Numberoftrainingsamples/3) _num_files_train = 3 #5 _num_images_train = _num_files_train * _images_per_file data_path = "D:/Iheya_n/HvassTutResults/2BatSCrNSe/NewResults/3to1ratio/{}Train/{}/".format(Numberoftrainingsamples,j) #data_path = cifar10.data_path data_path1 = "D:/Iheya_n/HvassTutResults/2BatSCrNSe/NewResults/3to1ratio/{}Train/".format(Numberoftrainingsamples) class_names = cifar10.load_class_names(data_path1) images_train, cls_train, labels_train = cifar10.load_training_data(data_path1,_num_images_train,data_path) images_test, cls_test, labels_test = cifar10.load_test_data(data_path1,data_path) openfile = open(data_path + "results.txt", "a") print("Size of:") print("- Training-set:\t\t{}".format(len(images_train))) print("- Test-set:\t\t{}".format(len(images_test))) openfile.write("size of:\n- Training-set:\t\t{}\n- Test-set:\t\t{}".format(len(images_train),len(images_test))) #save_dir = data_path + 'checkpoints/' save_dir = data_path + 'checkpoints/' if not os.path.exists(save_dir): os.makedirs(save_dir)
def main(): NUM_CLASSES = 10 dropout = 0.5 BATCH_SIZE = 128 NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN = 50000 INITIAL_LEARNING_RATE = 0.1 LEARNING_RATE_DECAY_FACTOR = 0.1 NUM_EPOCHS_PER_DECAY = 350.0 MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average. DISPLAY_FREQ = 20 TRAIN_OR_TEST = 1 # model_name = 'tmp_20160130.pkl' # model_name = 'data_sync/test20170203.pkl' #model_name = '20170205.pkl' model_name = './data/20170209.pkl' # model_name = 'data_sync/20170206.pkl' # model_name = 'test.pkl' # model_name = '../tf_official_docker/tmp.pkl' PREV_MODEL_EXIST = 1 # cls_train returns as an integer, labels is the array cifar10.maybe_download_and_extract() class_names = cifar10.load_class_names() images_train, cls_train, labels_train = cifar10.load_training_data() t_data = training_data(images_train, labels_train) DATA_CNT = len(images_train) NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE training_data_list = [] weights, biases = initialize_variables(PREV_MODEL_EXIST, model_name) x = tf.placeholder(tf.float32, [None, 32, 32, 3]) y = tf.placeholder(tf.float32, [None, NUM_CLASSES]) keep_prob = tf.placeholder(tf.float32) images = pre_process(x, TRAIN_OR_TEST) pred = cov_network(images, weights, biases, keep_prob) # print(pred) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(pred, y) loss_value = tf.reduce_mean(cross_entropy) correct_prediction = tf.equal(tf.argmax(pred, 1), tf.argmax(y, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) saver = tf.train.Saver() # global_step = tf.contrib.framework.get_or_create_global_step() num_batches_per_epoch = NUM_EXAMPLES_PER_EPOCH_FOR_TRAIN / BATCH_SIZE decay_steps = int(num_batches_per_epoch * NUM_EPOCHS_PER_DECAY) # Decay the learning rate exponentially based on the number of steps. # lr = tf.train.exponential_decay(INITIAL_LEARNING_RATE, # global_step, # decay_steps, # LEARNING_RATE_DECAY_FACTOR, # staircase=True) # # opt = tf.train.GradientDescentOptimizer(lr) # grads = opt.compute_gradients(loss_value) # # Apply gradients. # train_step = opt.apply_gradients(grads, global_step=global_step) train_step = tf.train.GradientDescentOptimizer( INITIAL_LEARNING_RATE).minimize(loss_value) # variable_averages = tf.train.ExponentialMovingAverage( # MOVING_AVERAGE_DECAY, global_step) # variables_averages_op = variable_averages.apply(tf.trainable_variables()) init = tf.global_variables_initializer() # Launch the graph with tf.Session() as sess: sess.run(init) # restore model if exists # if (os.path.isfile("tmp_20160130/model.meta")): # op = tf.train.import_meta_graph("tmp_20160130/model.meta") # op.restore(sess,tf.train.latest_checkpoint('tmp_20160130/')) # print ("model found and restored") start = time.time() if TRAIN_OR_TEST == 1: for i in range(0, 100000): (batch_x, batch_y) = t_data.feed_next_batch(BATCH_SIZE) train_acc, cross_en = sess.run([accuracy, loss_value], feed_dict={ x: batch_x, y: batch_y, keep_prob: 1.0 }) if (i % DISPLAY_FREQ == 0): print('This is the {}th iteration, time is {}'.format( i, time.time() - start)) print("accuracy is {} and cross entropy is {}".format( train_acc, cross_en)) if (i % (DISPLAY_FREQ * 50) == 0 and i != 0): save_pkl_model(weights, biases, model_name) # saver.save(sess, "tmp_20160130/model") print("saved the network") _ = sess.run(train_step, feed_dict={ x: batch_x, y: batch_y, keep_prob: dropout }) images_test, cls_test, labels_test = cifar10.load_test_data() test_acc = sess.run(accuracy, feed_dict={ x: images_test, y: labels_test, keep_prob: 1.0 }) # save_pkl_model(weights, biases, model_name) print("test accuracy is {}".format(test_acc))
def main(argv = None): NUM_CLASSES = 10 BATCH_SIZE = 128 INITIAL_LEARNING_RATE = 0.001 LEARNING_RATE_DECAY_FACTOR = 0.1 NUM_EPOCHS_PER_DECAY = 350.0 MOVING_AVERAGE_DECAY = 0.9999 # The decay to use for the moving average. DISPLAY_FREQ = 20 TRAIN = 0 TEST = 1 TRAIN_OR_TEST = 0 NUM_CHANNELS = 3 DOCKER = 0 if (argv == None): NUMBER_OF_CLUSTER = 8 else: NUMBER_OF_CLUSTER = argv mask_dir = './weights_log/' # base_model_name = './data/20170206.pkl' base_model_name = 'base.pkl' parent_dir = './' PREV_MODEL_EXIST = 1 (weights_mask,biases_mask)= initialize_weights_mask(0, parent_dir + 'masks/'+ base_model_name) cifar10.maybe_download_and_extract() class_names = cifar10.load_class_names() images_train, cls_train, labels_train = cifar10.load_training_data() images_test, cls_test, labels_test = cifar10.load_test_data() test_data = format_data(images_test, labels_test) DATA_CNT = len(images_train) NUMBER_OF_BATCH = DATA_CNT / BATCH_SIZE weights, biases = initialize_variables(PREV_MODEL_EXIST, parent_dir + 'weights/' + base_model_name) x = tf.placeholder(tf.float32, [None, 32, 32, 3]) y = tf.placeholder(tf.float32, [None, NUM_CLASSES]) keep_prob = tf.placeholder(tf.float32) images = pre_process(x, 0) pred = cov_network(images, weights, biases, keep_prob) cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits = pred, labels = y) loss_value = tf.reduce_mean(cross_entropy) correct_prediction = tf.equal(tf.argmax(pred,1), tf.argmax(y,1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) saver = tf.train.Saver() global_step = tf.contrib.framework.get_or_create_global_step() init = tf.global_variables_initializer() print('Graph launching ..') with tf.Session() as sess: sess.run(init) # prune the network keys = ['cov1', 'cov2', 'fc1', 'fc2', 'fc3'] for key in keys: sess.run(weights[key].assign(weights[key].eval()*weights_mask[key])) sess.run(biases[key].assign(biases[key].eval()*biases_mask[key])) print('quantisation process starts..') prune_info(weights, 0) NUMBER_OF_BATCH = 10000 / BATCH_SIZE t_acc = [] test_data = format_data(images_test, labels_test) for i in range(0,NUMBER_OF_BATCH): (batch_x, batch_y) = test_data.feed_next_batch(BATCH_SIZE) test_acc = sess.run(accuracy, feed_dict = { x: batch_x, y: batch_y, keep_prob: 1.0}) t_acc.append(test_acc) print("test accuracy is {}".format(t_acc)) test_acc = np.mean(t_acc) print('Pre quantisation model has an accuracy of {}'.format(test_acc)) print(78*'-') start = time.time() keys = ['cov1','cov2','fc1','fc2','fc3'] weights_val = {} centroids = {} cluster_index = {} weights_orgs = {} biases_orgs = {} for key in keys: weight_org = weights[key].eval() weight= weight_org.flatten() weight_val = weight[weight != 0] data = np.expand_dims(weight_val, axis = 1) print(np.shape(data)) # use kmeans to cluster # kmeans = KMeans(n_clusters= NUMBER_OF_CLUSTER, random_state=1).fit(data) kmeans = MiniBatchKMeans(n_clusters= NUMBER_OF_CLUSTER, random_state=0, init='k-means++').fit(data) centroid = kmeans.cluster_centers_ # add centroid value centroids[key] = centroid # add index value # indexs are stored in weight_org index = kmeans.labels_ + 1 for w in np.nditer(weight_org, op_flags=['readwrite']): if (w != 0): w[...] = kmeans.predict(w)+1 # sys.exit() cluster_index[key] = weight_org weights_orgs[key] = weights[key].eval() biases_orgs[key] = biases[key].eval() print('quantisation done') print(78*'-') print("test accuracy is {}".format(test_acc)) with open(parent_dir + 'weights/' + 'weightspt' +str(NUMBER_OF_CLUSTER)+'.pkl','wb') as f: pickle.dump((weights_orgs, biases_orgs, cluster_index,centroids),f)
def cifar10_reader(): from cifar10 import load_training_data, load_test_data, load_class_names train_x, _, train_y = load_training_data() test_x, _, test_y = load_test_data() class_name = load_class_names() return train_x, train_y, test_x, test_y, class_name
import cifar10 #from cifar10 import num_classes num_classes = 2 ##################################################################################### ##################################################################################### # DATA LOADING # ##################################################################################### ##################################################################################### cifar10.maybe_download_and_extract() class_names_load = cifar10.load_class_names() class_names_load images_train, cls_train_load, labels_train_load = cifar10.load_training_data([b'deer',b'horse']) images_test, cls_test_load, labels_test_load = cifar10.load_test_data([b'dog',b'truck']) images_test2, cls_test_load2, labels_test_load2 = cifar10.load_test_data([b'ship',b'frog']) images_test3, cls_test_load3, labels_test_load3 = cifar10.load_test_data([b'deer',b'horse']) # binarising classes, labels and class names cls_train,labels_train = class_binariser(cls_train_load),label_binariser(labels_train_load) cls_test,labels_test = class_binariser(cls_test_load),label_binariser(labels_test_load) cls_test2,labels_test2 = class_binariser(cls_test_load2),label_binariser(labels_test_load2) # to see if the classifier will work on this other set cls_test3,labels_test3 = class_binariser(cls_test_load3),label_binariser(labels_test_load3)
return image def pre_process(images, img_size_cropped, num_channels, training): # Use TensorFlow to loop over all the input images and call # the function above which takes a single image as input. images = tf.map_fn(lambda image: pre_process_image(image, img_size_cropped, num_channels, training), images) return images ## #load data cifar10.maybe_download_and_extract() class_names = cifar10.load_class_names() print(class_names) images_train, cls_train, labels_train = cifar10.load_training_data() images_test, cls_test, labels_test = cifar10.load_test_data() print("Size of:") print("- Training-set:\t\t{}".format(len(images_train))) print("- Test-set:\t\t{}".format(len(images_test))) img_size_cropped = 24 ## X = tf.placeholder(tf.float32, shape=[None, img_size, img_size, num_channels], name='X') images = pre_process(images=X, img_size_cropped=img_size_cropped, num_channels=num_channels,training=True)