def prune_least_important_features(prune_count):
model = FC_Net(layer_sizes=[784, 32, 32, 32, 32, 10])
state_dict = torch.load(file)
model.load_state_dict(state_dict)
trainer.model = model
print("\n prune the ", prune_count,
" least important features for all layers at once\n")
for name, module in model.named_modules():
if type(module) is Conv1dExtendable:
ncc = module.normalized_cross_correlation()
else:
continue
for _ in range(prune_count):
(min_val, min_idx) = torch.min(torch.abs(ncc), 0)
print("in ", name, ", prune feature number ", min_idx[0],
" with ncc ", min_val[0])
ncc = module.prune_feature(feature_number=min_idx[0])
# (sorted_val, sorted_idx) = torch.sort(torch.abs(ncc), dim=0, descending=False)
# feature_numbers = sorted_idx[0:prune_count]
# for i, feature_number in enumerate(feature_numbers):
# print("in ", name, ", prune feature number ", feature_number, " with ncc ", sorted_val[i])
# module.prune_feature(feature_number=feature_number)
test_loss, correct = trainer.test()
print("change of test loss: ", (test_loss / orig_test_loss))
print("")
def pruning_sweep_most_important(prune_count):
model = FC_Net(layer_sizes=[784, 32, 32, 32, 32, 10])
state_dict = torch.load(file)
model.load_state_dict(state_dict)
trainer.model = model
print(
"Prune the most important feature of each layer a given number of times without reloading the model"
)
for i in range(prune_count):
print("reduce most important feature (", i, ")")
for name, module in model.named_modules():
if type(module) is Conv1dExtendable:
ncc = module.normalized_cross_correlation()
else:
continue
(max_val, max_idx) = torch.max(torch.abs(ncc), 0)
(sorted_val, sorted_idx) = torch.sort(torch.abs(ncc),
dim=0,
descending=True)
feature_numbers = sorted_idx[0:1]
for i, feature_number in enumerate(feature_numbers):
print("in ", name, ", prune feature number ", feature_number,
" with ncc ", sorted_val[i])
module.prune_feature(feature_number=feature_number)
test_loss = trainer.test()
print("change of test loss: ", (test_loss / orig_test_loss))
print("")
def sort_importance_by_testing(module_name):
module = trainer.model.seq.__getattr__(module_name)
test_results = []
for feature_number in range(module.out_channels):
module = trainer.model.seq.__getattr__(module_name)
module.prune_feature(feature_number)
res = trainer.test()
test_results.append(res)
model = FC_Net(layer_sizes=[784, 32, 32, 32, 32, 10])
state_dict = torch.load(file)
model.load_state_dict(state_dict)
trainer.model = model
tr = torch.FloatTensor(test_results)
(sorted_val, sorted_idx) = torch.sort(torch.abs(tr), dim=0, descending=False)
print("indices of the features sorted by importance:")
print(sorted_idx)
return sorted_idx
import torch
import os
from models import FC_Net
from training import OptimizerMNIST
from expanding_modules import Conv1dExtendable
from logger import Logger
model = FC_Net(layer_sizes=[784, 4, 4, 10])
trainer = OptimizerMNIST(model,
epochs=20,
expand_interval=400,
log_interval=400,
expand_threshold=0.05,
prune_threshold=0.0,
expand_rate=8,
lr=0.0001,
weight_decay=0)
#name = str(model.layer_count) + "_layers_" + "_extend_" + str(trainer.extend_threshold) + "_prune_" + str(trainer.prune_threshold) + "_Adam"
name = str(model.layer_count) + "_layers_" + "_extend_" + str(
trainer.expand_rate) + "_interval_" + str(
trainer.expand_interval) + "_lr_" + str(trainer.lr) + "_min"
folder = "./experiments/FixedExpansionRate/" + name
#logger=Logger(folder)
#trainer.logger = logger
trainer.train()
torch.save(model.state_dict(),
folder + "/model_trained_for_" + str(trainer.epochs) + "_epochs")
import torch
import os
from models import FC_Net
from training import MNIST_Optimizer
from expanding_modules import Conv1dExtendable
model = FC_Net(layer_sizes=[784, 32, 32, 32, 32, 10])
trainer = MNIST_Optimizer(model, epochs=10)
file = "trained_model_784_4x32_10"
if os.path.exists(file):
print("load model")
state_dict = torch.load(file)
model.load_state_dict(state_dict)
else:
print("save model")
trainer.train()
torch.save(model.state_dict(), file)
print("initial test run:")
orig_test_loss, correct = trainer.test()
# trainer.extend_threshold = 10
# trainer.prune_threshold = 0.12
# trainer.extend_and_prune(0)
# prune_count = 20
#
# print("\n prune the ", prune_count, " most important features seperately\n ")
# for name, module in model.named_modules():
import torch
import os
from models import FC_Net, Conv_Net
from training import MNIST_Optimizer
from torch.autograd import Variable
from expanding_modules import Conv1dExtendable, Conv2dExtendable
model = FC_Net(layer_sizes=[784, 32, 32, 32, 32, 10])
#model = Conv_Net()
trainer = MNIST_Optimizer(model, epochs=10)
file = "trained_model_784_4x32_10"
if os.path.exists(file):
print("load model")
state_dict = torch.load(file)
model.load_state_dict(state_dict)
else:
print("save model")
trainer.train()
torch.save(model.state_dict(), file)
print("initial test run:")
orig_test_loss, correct = trainer.test()
test_img = Variable(torch.rand(28, 28))
print(model(test_img))
for name, module in model.named_modules():
if type(module) is Conv1dExtendable or type(module) is Conv2dExtendable:
ncc = module.normalized_cross_correlation()