def plot_figure4b_replica(): hist_iter = np.load("data/iterpr_data/iterpr_lenet_20perc.npz", allow_pickle=True)['histories'] hist_iter_reinit = np.load("data/iterpr_data/iterpr_lenet_20perc_reinit.npz", allow_pickle=True)['histories'] hist_os = np.load("data/ospr_data/OneShotPruningAcc_5trials_20epochs_20percOut.npz", allow_pickle=True)['histories'] hist_os_reinit = np.load("data/ospr_data/os_acc_reinit.npz", allow_pickle=True)['histories'] percentages, _ = generate_percentages([0.0, 1.0, 1.0, 1.0], 0.02) hist_iter = extract_final_accuracies(hist_iter) hist_iter_reinit = extract_final_accuracies(hist_iter_reinit) percentages_list = list() percentages_list.append(100.0) for i in range(len(percentages)): percentages_list.append(percentages[i][1]*100) plt.plot(percentages_list, hist_iter, label="Winning Ticket (Iterative)", color='b', marker='v') plt.plot(percentages_list, hist_iter_reinit, label="Random Reinit (Iterative)", color='k', marker='.', linestyle='--') plt.plot(percentages_list, hist_os, label="Winning Ticket (Oneshot)", color='g', marker='+') plt.plot(percentages_list, hist_os_reinit, label="Random Reinit (Oneshot)", color='r', marker='x', linestyle='--') plt.legend() plt.title("Accuracy at the end of training for different pruning methods") plt.xlabel("Percent of Weights Remaining") plt.ylabel("Accuracy at Iteration 20K (Test)") plt.xlim(left=100.5, right=1.5) plt.xscale('log') plt.grid() plt.xticks([100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75], [100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75]) plt.show()
def iterative_test_conv(settings, network_type=2, filename=""): percents, iterations = generate_percentages( [1.0, 1.0, 1.0], 0.02, settings['pruning_percentages']) histories = np.zeros(iterations + 1) es_epochs = np.zeros(iterations + 1) for trial in range(TRIALS): if network_type == 2: og_network = CONV2_NETWORK(settings) elif network_type == 4: og_network = CONV4_NETWORK(settings) elif network_type == 6: og_network = CONV6_NETWORK(settings) #Save initial weights of the original matrix init_weights = og_network.get_weights() #Train original Network mask = prune(og_network, 1.0, 1.0, 1.0) _, epoch = og_network.fit_batch(x_train, y_train, mask, init_weights, x_test, y_test) es_epochs[0] += epoch #Evaluate original network and save results _, test_acc = og_network.evaluate_model(x_test, y_test) histories[0] += test_acc #Prune the network for x amount of iterations, evaulate each iteration and save results for i in range(0, iterations): print("Conv %: " + str(percents[i][0]) + ", Dense %: " + str(percents[i][1]) + ", Output %: " + str(percents[i][2])) mask = prune(og_network, percents[i][0], percents[i][1], percents[i][2]) #for w in masked_weights: # print(np.count_nonzero(w==0)/np.size(w)) if network_type == 2: pruned_network = CONV2_NETWORK(settings) elif network_type == 4: pruned_network = CONV4_NETWORK(settings) elif network_type == 6: pruned_network = CONV6_NETWORK(settings) _, epoch = pruned_network.fit_batch(x_train, y_train, mask, init_weights, x_test, y_test) _, test_acc = pruned_network.evaluate_model(x_test, y_test) histories[i + 1] += test_acc es_epochs[i + 1] += epoch og_network = pruned_network filename += "_trial-" + str(trial + 1) print(histories) print(es_epochs) np.savez(filename + ".npz", histories=histories, es_epochs=es_epochs) return histories, es_epochs
def plot_figure4a_replica(): hist_iter = np.load("data/iterpr_data/iterpr_lenet_20perc_es.npz", allow_pickle=True)['histories'] hist_iter_reinit = np.load("data/iterpr_data/iterpr_lenet_20perc_reinit_es.npz", allow_pickle=True)['histories'] hist_iter_epochs = np.load("data/iterpr_data/iterpr_lenet_20perc_es.npz", allow_pickle=True)['es_epochs'] hist_iter_reinit_epochs = np.load("data/iterpr_data/iterpr_lenet_20perc_reinit_es.npz", allow_pickle=True)['es_epochs'] hist_os = np.load("data/ospr_data/OneShotPruningAcc_5trials_50epochs_20perc_ES.npz", allow_pickle=True)['histories'] hist_os_reinit = np.load("data/ospr_data/OneShotPruningAcc_5trials_50epochs_20perc_ES_rand.npz", allow_pickle=True)['histories'] hist_os_epochs = np.load("data/ospr_data/OneShotPruningEpochs_5trials_50epochs_ES.npz", allow_pickle=True)['histories'] hist_os_reinit_epochs = np.load("data/ospr_data/OneShotPruningEpochs_5trials_50epochs_ES_rand.npz", allow_pickle=True)['histories'] percentages, _ = generate_percentages([0.0, 1.0, 1.0, 1.0], 0.02) percentages_list = list() percentages_list.append(100.0) for i in range(len(percentages)): percentages_list.append(percentages[i][1]*100) hist_iter, hist_iter_reinit, hist_iter_epochs, hist_iter_reinit_epochs = process_es_data(hist_iter, hist_iter_reinit, hist_iter_epochs, hist_iter_reinit_epochs) # Plots Early-Stop iteration (Figure 4ai) plt.plot(percentages_list, hist_iter_epochs, label="Winning Ticket (Iterative)", color='b', marker='v') plt.plot(percentages_list, hist_iter_reinit_epochs, label="Random Reinit (Iterative)", color='k', marker='.', linestyle='--') plt.plot(percentages_list, hist_os_epochs, label="Winning Ticket (Oneshot)", color='g', marker='+') plt.plot(percentages_list, hist_os_reinit_epochs, label="Random Reinit (Oneshot)", color='r', marker='x', linestyle='--') plt.legend() plt.title("Early stopping iteration for all pruning methods") plt.xlabel("Percent of Weights Remaining") plt.ylabel("Early-Stop Epoch (Val.)") plt.xlim(left=100.5, right=1.5) plt.xscale('log') plt.grid() plt.xticks([100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75], [100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75]) plt.show() # Plots accuracy at early-stop (Figure 4aii) plt.plot(percentages_list, hist_iter, label="Winning Ticket (Iterative)", color='b', marker='v') plt.plot(percentages_list, hist_iter_reinit, label="Random Reinit (Iterative)", color='k', marker='.', linestyle='--') plt.plot(percentages_list, hist_os, label="Winning Ticket (Oneshot)", color='g', marker='+') plt.plot(percentages_list, hist_os_reinit, label="Random Reinit (Oneshot)", color='r', marker='x', linestyle='--') plt.legend() plt.title("Accuracy at Early-stop for all pruning methods") plt.xlabel("Percent of Weights Remaining") plt.ylabel("Accuracy at Early-Stop (Test)") plt.xlim(left=100.5, right=1.5) plt.xscale('log') plt.grid() plt.xticks([100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75], [100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75]) plt.show()
def iterative_pruning_experiment(): """ - For multiple trials: - Create the original network - Train original network, save results - For multiple iterations: - Create mask with the pruning precentages generated by generate_percentages function in tools.py - Prune the network with the pruning percentages - Train the pruned network, save results - Set the original network to be the pruned network """ trials = SETTINGS['trials'] percents = [0.0, 1.0, 1.0, 1.0] percentages, iterations = generate_percentages(percents, SETTINGS['lower_bound']) histories = np.zeros((iterations + 1, SETTINGS['n_epochs'])) #es_epochs = np.zeros((trials, iterations+1)) for k in range(0, trials): print("TRIAL " + str(k + 1) + "/" + str(trials)) og_network = FC_NETWORK(use_earlyStopping=SETTINGS['use_es']) init_weights = og_network.weights_init mask = prune(og_network, percents) acc_history, _ = og_network.fit_batch(x_train, y_train, mask, init_weights, SETTINGS, x_test, y_test) histories[iterations] += np.asarray(acc_history) #es_epochs[k, iterations] = epoch for i in range(0, iterations): S = percentages[i] print("Prune iteration: " + str(i + 1) + "/" + str(iterations) + ", S: " + str(S)) print("Creating the pruned network") mask = prune(og_network, S) pruned_network = FC_NETWORK(use_earlyStopping=SETTINGS['use_es']) acc_history, _ = pruned_network.fit_batch(x_train, y_train, mask, init_weights, SETTINGS, x_test, y_test) histories[i] += np.asarray(acc_history) #es_epochs[k, i] = epoch og_network = pruned_network histories = histories / trials #es_epochs = float(es_epochs/trials) np.savez("data/iterpr_lenet_20perc.npz", histories=histories)
def one_shot_pruning_experiment(): """ - For multiple trials: - Train original network and evaluate results - Find mask depending on weights of original network - Apply mask to a new network and disable corresponding weights - Train the new pruned network and evaluate results """ percentages, iterations = generate_percentages([0.0, 1.0, 1.0, 1.0], SETTINGS['lower_bound']) #print(percentages) #print(iterations) trials = SETTINGS['trials'] og_networks = list() tot_acc = np.zeros(iterations + 1) tot_loss = np.zeros(iterations + 1) tot_epoch = np.zeros(iterations + 1) # Training and evaluating the unpruned network over multiple trials print("Training and Evaluating OG networks") percents = [0.0, 1.0, 1.0, 1.0] for i in range(0, trials): print("TRIAL " + str(i + 1) + "/" + str(trials)) og_networks.append(FC_NETWORK(use_earlyStopping=SETTINGS['use_es'])) mask = prune(og_networks[i], percents) _, epoch = og_networks[i].fit_batch(x_train, y_train, mask, og_networks[i].weights_init, SETTINGS, x_test, y_test) test_loss, test_acc = og_networks[i].evaluate_model(x_test, y_test) tot_acc[0] += test_acc tot_loss[0] += test_loss tot_epoch[0] += epoch print(epoch) tot_acc[0] = float(tot_acc[0] / trials) tot_loss[0] = float(tot_loss[0] / trials) tot_epoch[0] = float(tot_epoch[0] / trials) # Training and evaluating pruned networks of different pruning rates over multiple trials for j in range(1, iterations + 1): print("Training and Evaluating pruned networks, iteration: " + str(j) + "/" + str(iterations)) print("Percentage: " + str(percentages[j - 1])) for og in og_networks: mask = prune(og, percentages[j - 1]) pruned_network = FC_NETWORK(use_earlyStopping=SETTINGS['use_es']) _, epoch = pruned_network.fit_batch(x_train, y_train, mask, og.weights_init, SETTINGS, x_test, y_test) print(epoch) test_loss, test_acc = pruned_network.evaluate_model(x_test, y_test) tot_acc[j] += test_acc tot_loss[j] += test_loss tot_epoch[j] += epoch tot_acc[j] = float(tot_acc[j] / trials) tot_loss[j] = float(tot_loss[j] / trials) tot_epoch[j] = float(tot_epoch[j] / trials) print(tot_epoch) print(tot_acc) print(tot_loss) print(tot_epoch)
def plot_figure3_replica(): histories = np.load("data/iterpr_data/iterpr_lenet_20perc.npz", allow_pickle=True)['histories'] histories_reinit = np.load("data/iterpr_data/iterpr_lenet_20perc_reinit.npz", allow_pickle=True)['histories'] percentages, _ = generate_percentages([0.0, 1.0, 1.0, 1.0], 0.02) markers = { 0: '8', 1: 'o', 2: 'v', 3: 'x', 4: 's', 5: '+', 6: '.' } #Plot of structures: 100.0, 51.3, 21.1 (Figure 3i) plt.plot(range(0, len(histories[0])*1000, 1000), histories[len(histories)-1], label="100.0", marker=markers[0]) for idx, i in enumerate([2, 6]): plt.plot(range(0,len(histories[i])*1000,1000), histories[i], label=str(np.around(percentages[i][1]*100, decimals=3)), marker=markers[idx+1]) plt.legend() plt.grid() plt.title("Test Accuracy on LeNet-5 with different pruning rates (using iterative pruning)") plt.xlabel("Training Iterations") plt.ylabel("Test Accuracy") plt.show() #Plot of structures: 100.0, 51.3, 21.1, 7.0, 3.6, 1.9 (Figure 3ii) plt.plot(range(0, len(histories[0])*1000, 1000), histories[len(histories)-1], label="100.0", marker=markers[0]) for idx, i in enumerate([2, 6, 11, 14, 17]): plt.plot(range(0,len(histories[i])*1000,1000), histories[i], label=str(np.around(percentages[i][1]*100, decimals=3)), marker=markers[idx+1]) plt.legend() plt.grid() plt.title("Test Accuracy on LeNet-5 with different pruning rates (using iterative pruning)") plt.xlabel("Training Iterations") plt.ylabel("Test Accuracy") plt.show() #Plot of structures: 100.0, 51.3, 21.1, 51.3 (reinit), 21.1 (reinit) (Figure 3iii) plt.plot(range(0, len(histories[0])*1000, 1000), histories[len(histories)-1], label="100.0", marker=markers[0]) for idx, i in enumerate([2, 6]): plt.plot(range(0,len(histories[i])*1000,1000), histories[i], label=str(np.around(percentages[i][1]*100, decimals=3)), marker=markers[idx+1]) for idx, i in enumerate([2, 6]): plt.plot(range(0, len(histories[i])*1000, 1000), histories_reinit[i], label=str(np.around(percentages[i][1]*100, decimals=3)) + "(reinit)", marker=markers[idx+1], linestyle='dashed') plt.legend() plt.grid() plt.title("Test Accuracy on LeNet-5 with different pruning rates (using iterative pruning)") plt.xlabel("Training Iterations") plt.ylabel("Test Accuracy") plt.show()
import numpy as np import matplotlib.pyplot as plt from tools import generate_percentages from constants import SETTINGS_CONV2, SETTINGS_CONV4, SETTINGS_CONV6 # Create consistent mapping for all plots percentages24, _ = generate_percentages([1.0, 1.0, 1.0], 0.02, SETTINGS_CONV2['pruning_percentages']) percentages_list_conv24 = list() percentages_list_conv24.append(100.0) for i in range(len(percentages24)): percentages_list_conv24.append(percentages24[i][1] * 100) percentages6, _ = generate_percentages([1.0, 1.0, 1.0], 0.02, SETTINGS_CONV6['pruning_percentages']) percentages_list_conv6 = list() percentages_list_conv6.append(100.0) for i in range(len(percentages6)): percentages_list_conv6.append(percentages6[i][1] * 100) def plot_dropout(): """ Plots network accuracy and early-stop epoch for networks with and without dropout applied. """ conv2_dropout_hist = np.load("data/conv2_rand-False_bn-False_dropout.npz", allow_pickle=True)['histories'] conv2_dropout_epochs = np.load(