def plot_figure4b_replica():
hist_iter = np.load("data/iterpr_data/iterpr_lenet_20perc.npz", allow_pickle=True)['histories']
hist_iter_reinit = np.load("data/iterpr_data/iterpr_lenet_20perc_reinit.npz", allow_pickle=True)['histories']
hist_os = np.load("data/ospr_data/OneShotPruningAcc_5trials_20epochs_20percOut.npz", allow_pickle=True)['histories']
hist_os_reinit = np.load("data/ospr_data/os_acc_reinit.npz", allow_pickle=True)['histories']
percentages, _ = generate_percentages([0.0, 1.0, 1.0, 1.0], 0.02)
hist_iter = extract_final_accuracies(hist_iter)
hist_iter_reinit = extract_final_accuracies(hist_iter_reinit)
percentages_list = list()
percentages_list.append(100.0)
for i in range(len(percentages)):
percentages_list.append(percentages[i][1]*100)
plt.plot(percentages_list, hist_iter, label="Winning Ticket (Iterative)", color='b', marker='v')
plt.plot(percentages_list, hist_iter_reinit, label="Random Reinit (Iterative)", color='k', marker='.', linestyle='--')
plt.plot(percentages_list, hist_os, label="Winning Ticket (Oneshot)", color='g', marker='+')
plt.plot(percentages_list, hist_os_reinit, label="Random Reinit (Oneshot)", color='r', marker='x', linestyle='--')
plt.legend()
plt.title("Accuracy at the end of training for different pruning methods")
plt.xlabel("Percent of Weights Remaining")
plt.ylabel("Accuracy at Iteration 20K (Test)")
plt.xlim(left=100.5, right=1.5)
plt.xscale('log')
plt.grid()
plt.xticks([100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75], [100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75])
plt.show()
def iterative_test_conv(settings, network_type=2, filename=""):
percents, iterations = generate_percentages(
[1.0, 1.0, 1.0], 0.02, settings['pruning_percentages'])
histories = np.zeros(iterations + 1)
es_epochs = np.zeros(iterations + 1)
for trial in range(TRIALS):
if network_type == 2:
og_network = CONV2_NETWORK(settings)
elif network_type == 4:
og_network = CONV4_NETWORK(settings)
elif network_type == 6:
og_network = CONV6_NETWORK(settings)
#Save initial weights of the original matrix
init_weights = og_network.get_weights()
#Train original Network
mask = prune(og_network, 1.0, 1.0, 1.0)
_, epoch = og_network.fit_batch(x_train, y_train, mask, init_weights,
x_test, y_test)
es_epochs[0] += epoch
#Evaluate original network and save results
_, test_acc = og_network.evaluate_model(x_test, y_test)
histories[0] += test_acc
#Prune the network for x amount of iterations, evaulate each iteration and save results
for i in range(0, iterations):
print("Conv %: " + str(percents[i][0]) + ", Dense %: " +
str(percents[i][1]) + ", Output %: " + str(percents[i][2]))
mask = prune(og_network, percents[i][0], percents[i][1],
percents[i][2])
#for w in masked_weights:
# print(np.count_nonzero(w==0)/np.size(w))
if network_type == 2:
pruned_network = CONV2_NETWORK(settings)
elif network_type == 4:
pruned_network = CONV4_NETWORK(settings)
elif network_type == 6:
pruned_network = CONV6_NETWORK(settings)
_, epoch = pruned_network.fit_batch(x_train, y_train, mask,
init_weights, x_test, y_test)
_, test_acc = pruned_network.evaluate_model(x_test, y_test)
histories[i + 1] += test_acc
es_epochs[i + 1] += epoch
og_network = pruned_network
filename += "_trial-" + str(trial + 1)
print(histories)
print(es_epochs)
np.savez(filename + ".npz", histories=histories, es_epochs=es_epochs)
return histories, es_epochs
def plot_figure4a_replica():
hist_iter = np.load("data/iterpr_data/iterpr_lenet_20perc_es.npz", allow_pickle=True)['histories']
hist_iter_reinit = np.load("data/iterpr_data/iterpr_lenet_20perc_reinit_es.npz", allow_pickle=True)['histories']
hist_iter_epochs = np.load("data/iterpr_data/iterpr_lenet_20perc_es.npz", allow_pickle=True)['es_epochs']
hist_iter_reinit_epochs = np.load("data/iterpr_data/iterpr_lenet_20perc_reinit_es.npz", allow_pickle=True)['es_epochs']
hist_os = np.load("data/ospr_data/OneShotPruningAcc_5trials_50epochs_20perc_ES.npz", allow_pickle=True)['histories']
hist_os_reinit = np.load("data/ospr_data/OneShotPruningAcc_5trials_50epochs_20perc_ES_rand.npz", allow_pickle=True)['histories']
hist_os_epochs = np.load("data/ospr_data/OneShotPruningEpochs_5trials_50epochs_ES.npz", allow_pickle=True)['histories']
hist_os_reinit_epochs = np.load("data/ospr_data/OneShotPruningEpochs_5trials_50epochs_ES_rand.npz", allow_pickle=True)['histories']
percentages, _ = generate_percentages([0.0, 1.0, 1.0, 1.0], 0.02)
percentages_list = list()
percentages_list.append(100.0)
for i in range(len(percentages)):
percentages_list.append(percentages[i][1]*100)
hist_iter, hist_iter_reinit, hist_iter_epochs, hist_iter_reinit_epochs = process_es_data(hist_iter,
hist_iter_reinit, hist_iter_epochs, hist_iter_reinit_epochs)
# Plots Early-Stop iteration (Figure 4ai)
plt.plot(percentages_list, hist_iter_epochs, label="Winning Ticket (Iterative)", color='b', marker='v')
plt.plot(percentages_list, hist_iter_reinit_epochs, label="Random Reinit (Iterative)", color='k', marker='.', linestyle='--')
plt.plot(percentages_list, hist_os_epochs, label="Winning Ticket (Oneshot)", color='g', marker='+')
plt.plot(percentages_list, hist_os_reinit_epochs, label="Random Reinit (Oneshot)", color='r', marker='x', linestyle='--')
plt.legend()
plt.title("Early stopping iteration for all pruning methods")
plt.xlabel("Percent of Weights Remaining")
plt.ylabel("Early-Stop Epoch (Val.)")
plt.xlim(left=100.5, right=1.5)
plt.xscale('log')
plt.grid()
plt.xticks([100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75], [100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75])
plt.show()
# Plots accuracy at early-stop (Figure 4aii)
plt.plot(percentages_list, hist_iter, label="Winning Ticket (Iterative)", color='b', marker='v')
plt.plot(percentages_list, hist_iter_reinit, label="Random Reinit (Iterative)", color='k', marker='.', linestyle='--')
plt.plot(percentages_list, hist_os, label="Winning Ticket (Oneshot)", color='g', marker='+')
plt.plot(percentages_list, hist_os_reinit, label="Random Reinit (Oneshot)", color='r', marker='x', linestyle='--')
plt.legend()
plt.title("Accuracy at Early-stop for all pruning methods")
plt.xlabel("Percent of Weights Remaining")
plt.ylabel("Accuracy at Early-Stop (Test)")
plt.xlim(left=100.5, right=1.5)
plt.xscale('log')
plt.grid()
plt.xticks([100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75], [100, 51.4, 26.5, 13.7, 7.1, 3.7, 1.75])
plt.show()
def iterative_pruning_experiment():
"""
- For multiple trials:
- Create the original network
- Train original network, save results
- For multiple iterations:
- Create mask with the pruning precentages generated by generate_percentages function in tools.py
- Prune the network with the pruning percentages
- Train the pruned network, save results
- Set the original network to be the pruned network
"""
trials = SETTINGS['trials']
percents = [0.0, 1.0, 1.0, 1.0]
percentages, iterations = generate_percentages(percents,
SETTINGS['lower_bound'])
histories = np.zeros((iterations + 1, SETTINGS['n_epochs']))
#es_epochs = np.zeros((trials, iterations+1))
for k in range(0, trials):
print("TRIAL " + str(k + 1) + "/" + str(trials))
og_network = FC_NETWORK(use_earlyStopping=SETTINGS['use_es'])
init_weights = og_network.weights_init
mask = prune(og_network, percents)
acc_history, _ = og_network.fit_batch(x_train, y_train, mask,
init_weights, SETTINGS, x_test,
y_test)
histories[iterations] += np.asarray(acc_history)
#es_epochs[k, iterations] = epoch
for i in range(0, iterations):
S = percentages[i]
print("Prune iteration: " + str(i + 1) + "/" + str(iterations) +
", S: " + str(S))
print("Creating the pruned network")
mask = prune(og_network, S)
pruned_network = FC_NETWORK(use_earlyStopping=SETTINGS['use_es'])
acc_history, _ = pruned_network.fit_batch(x_train, y_train, mask,
init_weights, SETTINGS,
x_test, y_test)
histories[i] += np.asarray(acc_history)
#es_epochs[k, i] = epoch
og_network = pruned_network
histories = histories / trials
#es_epochs = float(es_epochs/trials)
np.savez("data/iterpr_lenet_20perc.npz", histories=histories)
def one_shot_pruning_experiment():
"""
- For multiple trials:
- Train original network and evaluate results
- Find mask depending on weights of original network
- Apply mask to a new network and disable corresponding weights
- Train the new pruned network and evaluate results
"""
percentages, iterations = generate_percentages([0.0, 1.0, 1.0, 1.0],
SETTINGS['lower_bound'])
#print(percentages)
#print(iterations)
trials = SETTINGS['trials']
og_networks = list()
tot_acc = np.zeros(iterations + 1)
tot_loss = np.zeros(iterations + 1)
tot_epoch = np.zeros(iterations + 1)
# Training and evaluating the unpruned network over multiple trials
print("Training and Evaluating OG networks")
percents = [0.0, 1.0, 1.0, 1.0]
for i in range(0, trials):
print("TRIAL " + str(i + 1) + "/" + str(trials))
og_networks.append(FC_NETWORK(use_earlyStopping=SETTINGS['use_es']))
mask = prune(og_networks[i], percents)
_, epoch = og_networks[i].fit_batch(x_train, y_train, mask,
og_networks[i].weights_init,
SETTINGS, x_test, y_test)
test_loss, test_acc = og_networks[i].evaluate_model(x_test, y_test)
tot_acc[0] += test_acc
tot_loss[0] += test_loss
tot_epoch[0] += epoch
print(epoch)
tot_acc[0] = float(tot_acc[0] / trials)
tot_loss[0] = float(tot_loss[0] / trials)
tot_epoch[0] = float(tot_epoch[0] / trials)
# Training and evaluating pruned networks of different pruning rates over multiple trials
for j in range(1, iterations + 1):
print("Training and Evaluating pruned networks, iteration: " + str(j) +
"/" + str(iterations))
print("Percentage: " + str(percentages[j - 1]))
for og in og_networks:
mask = prune(og, percentages[j - 1])
pruned_network = FC_NETWORK(use_earlyStopping=SETTINGS['use_es'])
_, epoch = pruned_network.fit_batch(x_train, y_train, mask,
og.weights_init, SETTINGS,
x_test, y_test)
print(epoch)
test_loss, test_acc = pruned_network.evaluate_model(x_test, y_test)
tot_acc[j] += test_acc
tot_loss[j] += test_loss
tot_epoch[j] += epoch
tot_acc[j] = float(tot_acc[j] / trials)
tot_loss[j] = float(tot_loss[j] / trials)
tot_epoch[j] = float(tot_epoch[j] / trials)
print(tot_epoch)
print(tot_acc)
print(tot_loss)
print(tot_epoch)
def plot_figure3_replica():
histories = np.load("data/iterpr_data/iterpr_lenet_20perc.npz", allow_pickle=True)['histories']
histories_reinit = np.load("data/iterpr_data/iterpr_lenet_20perc_reinit.npz", allow_pickle=True)['histories']
percentages, _ = generate_percentages([0.0, 1.0, 1.0, 1.0], 0.02)
markers = {
0: '8',
1: 'o',
2: 'v',
3: 'x',
4: 's',
5: '+',
6: '.'
}
#Plot of structures: 100.0, 51.3, 21.1 (Figure 3i)
plt.plot(range(0, len(histories[0])*1000, 1000), histories[len(histories)-1], label="100.0", marker=markers[0])
for idx, i in enumerate([2, 6]):
plt.plot(range(0,len(histories[i])*1000,1000), histories[i],
label=str(np.around(percentages[i][1]*100, decimals=3)), marker=markers[idx+1])
plt.legend()
plt.grid()
plt.title("Test Accuracy on LeNet-5 with different pruning rates (using iterative pruning)")
plt.xlabel("Training Iterations")
plt.ylabel("Test Accuracy")
plt.show()
#Plot of structures: 100.0, 51.3, 21.1, 7.0, 3.6, 1.9 (Figure 3ii)
plt.plot(range(0, len(histories[0])*1000, 1000), histories[len(histories)-1], label="100.0", marker=markers[0])
for idx, i in enumerate([2, 6, 11, 14, 17]):
plt.plot(range(0,len(histories[i])*1000,1000), histories[i],
label=str(np.around(percentages[i][1]*100, decimals=3)), marker=markers[idx+1])
plt.legend()
plt.grid()
plt.title("Test Accuracy on LeNet-5 with different pruning rates (using iterative pruning)")
plt.xlabel("Training Iterations")
plt.ylabel("Test Accuracy")
plt.show()
#Plot of structures: 100.0, 51.3, 21.1, 51.3 (reinit), 21.1 (reinit) (Figure 3iii)
plt.plot(range(0, len(histories[0])*1000, 1000), histories[len(histories)-1], label="100.0", marker=markers[0])
for idx, i in enumerate([2, 6]):
plt.plot(range(0,len(histories[i])*1000,1000), histories[i],
label=str(np.around(percentages[i][1]*100, decimals=3)), marker=markers[idx+1])
for idx, i in enumerate([2, 6]):
plt.plot(range(0, len(histories[i])*1000, 1000), histories_reinit[i],
label=str(np.around(percentages[i][1]*100, decimals=3)) + "(reinit)", marker=markers[idx+1], linestyle='dashed')
plt.legend()
plt.grid()
plt.title("Test Accuracy on LeNet-5 with different pruning rates (using iterative pruning)")
plt.xlabel("Training Iterations")
plt.ylabel("Test Accuracy")
plt.show()
import numpy as np
import matplotlib.pyplot as plt
from tools import generate_percentages
from constants import SETTINGS_CONV2, SETTINGS_CONV4, SETTINGS_CONV6
# Create consistent mapping for all plots
percentages24, _ = generate_percentages([1.0, 1.0, 1.0], 0.02,
SETTINGS_CONV2['pruning_percentages'])
percentages_list_conv24 = list()
percentages_list_conv24.append(100.0)
for i in range(len(percentages24)):
percentages_list_conv24.append(percentages24[i][1] * 100)
percentages6, _ = generate_percentages([1.0, 1.0, 1.0], 0.02,
SETTINGS_CONV6['pruning_percentages'])
percentages_list_conv6 = list()
percentages_list_conv6.append(100.0)
for i in range(len(percentages6)):
percentages_list_conv6.append(percentages6[i][1] * 100)
def plot_dropout():
"""
Plots network accuracy and early-stop epoch for networks
with and without dropout applied.
"""
conv2_dropout_hist = np.load("data/conv2_rand-False_bn-False_dropout.npz",
allow_pickle=True)['histories']
conv2_dropout_epochs = np.load(