def prune(args, model, percent, train_loader, val_loader, hyperparams,
df_column_entry_dict):
"""
Uses parameter pruning to remove connections from the model that are least relevant for the neurons.
The concrete procedure is the traversal of the models modules, module by module (https://arxiv.org/abs/1506.02626).
# todo elaborate on the CONCRETE pruning procedure, especially when implementing alternatives or before varying this
Code from https://github.com/larry0123du/PyTorch-Deep-Compression
Code explained at https://jacobgil.github.io/deeplearning/pruning-deep-learning
:param model: the actual model (Module subclass), not a path, not just the weights
:return: the saved model's path
"""
# Set additional parameters required for pruning.
# todo future work: might want to include all of these in args only, instead of passing arguments in two parameters, args and hyperparams
hyperparams['topk'] = [1, 5] # Top k precision metrics
hyperparams['interval'] = int(args.prune_epochs) # checkpointing interval
hyperparams['momentum'] = 0.9
hyperparams['weight_decay'] = 0.005
torch.cuda.empty_cache()
print("emptied cache\n")
print_memory_metrics("start of pruning", df_column_entry_dict)
start_mem_measurement()
start = time.time()
iter_prune(args=args,
train_loader=train_loader,
val_loader=val_loader,
the_model=model,
stop_percent=percent,
df_column_entry_dict=df_column_entry_dict,
**hyperparams)
time_elapse = time.time() - start
event = 'iterative pruning'
formatted_time = str(timedelta(seconds=time_elapse))
df_column_entry_dict['Time measurement at ' + event + ' [s]'] = time_elapse
print("\n" + event + " took " + formatted_time + " seconds\n")
event = "end of pruning"
stop_mem_measurement(event, df_column_entry_dict)
print_memory_metrics(event, df_column_entry_dict)
def test(test_loader, model, criterion, loggers, activations_collectors, args):
"""Model Test"""
msglogger.info('--- test ---------------------')
if activations_collectors is None:
activations_collectors = create_activation_stats_collectors(
model, None)
with collectors_context(activations_collectors["test"]) as collectors:
df_column_entry_dict = {}
event = "validation"
if torch.cuda.is_available():
print_memory_metrics("before " + event, df_column_entry_dict)
start_mem_measurement()
start = time.time()
top1, top5, lossses = _validate(test_loader, model, criterion, loggers,
args)
time_elapse = time.time() - start
df_column_entry_dict['Time measurement at ' + event +
' [s]'] = time_elapse
stop_mem_measurement(event, df_column_entry_dict)
if torch.cuda.is_available():
print_memory_metrics("after " + event, df_column_entry_dict)
import pandas as pd
frame = pd.DataFrame([list(df_column_entry_dict.values())],
columns=list(df_column_entry_dict.keys()))
frame.to_excel(DISTILLER_PATH + msglogger.logdir + ".xlsx",
index=False)
distiller.log_activation_statsitics(-1,
"test",
loggers,
collector=collectors['sparsity'])
save_collectors_data(collectors, msglogger.logdir)
return top1, top5, lossses
def iter_prune(args,
train_loader,
val_loader,
the_model=None,
stop_percent=None,
df_column_entry_dict=None,
**kwargs):
topk = tuple(sorted(kwargs['topk'])) # sort in ascending order
epoch = 1
best_loss = sys.maxsize
assert the_model is not None
model = the_model # pass actual model from parameters
import DeepHyperX.models
# only get optimizer, not model again
_, optimizer, _, _ = DeepHyperX.models.get_model(args.model, **kwargs)
# if args.restore:
# checkpoint = load_checkpoint(args.restore)
# model.load_state_dict(checkpoint['state_dict'])
# epoch = checkpoint['epoch'] % args.epoch + 1
# best_loss = checkpoint['best_loss']
# try:
# optimizer.load_state_dict(checkpoint['optimizer'])
# except Exception as e:
# raise e
print(model.__class__)
criterion = nn.CrossEntropyLoss()
if args.cuda is not None:
args.cuda = 1 # in the main program, 0 specifies the 0th device, whereas in the pruning program, cuda==0 means cuda is disabled. This is a fix for that
if args.cuda:
model = nn.DataParallel(model).cuda()
criterion = criterion.cuda()
compressor = Compressor(model, cuda=args.cuda)
model.train()
pct_pruned = 0.0
scores = [AverageMeter() for _ in topk]
val_scores = [0.0 for _ in topk]
end_next_turn_flag = False
while True:
try:
if end_next_turn_flag:
break
if epoch == 1:
start = time.time()
new_pct_pruned = compressor.prune(args.alpha, model=args.model)
time_elapse = time.time() - start
event = 'compressor.prune'
formatted_time = str(timedelta(seconds=time_elapse))
df_column_entry_dict['Time measurement at ' + event +
' [s]'] = time_elapse
print("\n" + event + " took " + formatted_time + " seconds\n")
print_memory_metrics("after compressor.prune",
df_column_entry_dict)
print('Pruned %.3f %%' % (100 * new_pct_pruned))
start = time.time()
top_accs = validate(model,
val_loader,
topk,
cuda=args.cuda,
df_column_entry_dict=df_column_entry_dict)
time_elapse = time.time() - start
event = 'validate after compressor.prune'
formatted_time = str(timedelta(seconds=time_elapse))
df_column_entry_dict['Time measurement at ' + event +
' [s]'] = time_elapse
print("\n" + event + " took " + formatted_time + " seconds\n")
print_memory_metrics(
"after validation following compressor.prune",
df_column_entry_dict)
# Stopping criterion
if stop_percent == None:
if new_pct_pruned - pct_pruned <= 0.001 and converged(
val_scores, top_accs):
print(
"Pruning percentage increased by < 0.001 and no score has become better than 0.001, quitting pruning...\n"
)
torch.save(
model.state_dict(), DEEPHYPERX_PATH_LINUX +
"outputs/Experiments/" + "pruneMe/" + args.model +
"_alpha" + args.alpha + "_" +
(kwargs['former_technique'] if
kwargs['former_technique'] is not None else "") +
"_" +
(kwargs['former_components'] if
kwargs['former_components'] is not None else ""))
break
elif new_pct_pruned - pct_pruned <= 0.001:
print(
"Pruning percentage increased by < 0.001, quitting pruning...\n"
)
torch.save(
model.state_dict(), DEEPHYPERX_PATH_LINUX +
"outputs/Experiments/" + "pruneMe/" + args.model +
"_alpha" + args.alpha + "_" +
(kwargs['former_technique'] if
kwargs['former_technique'] is not None else "") +
"_" +
(kwargs['former_components'] if
kwargs['former_components'] is not None else ""))
break
else:
if new_pct_pruned * 100 > stop_percent:
print("Desired pruning percentage of " +
str(stop_percent) +
" reached, the new percentage would have been " +
str(new_pct_pruned * 100) +
", quitting pruning...\n")
end_next_turn_flag = True
pct_pruned = new_pct_pruned
val_scores = top_accs
for e in range(epoch, int(args.prune_epochs) + 1):
for i, (input, label) in enumerate(train_loader, 0):
input, label = Variable(input), Variable(label)
if args.cuda:
input, label = input.cuda(), label.cuda()
optimizer.zero_grad()
output = model(input)
precisions = accuracy(output, label, topk)
for i, s in enumerate(scores):
s.update(precisions[i][0], input.size(0))
loss = criterion(output, label)
loss.backward()
compressor.set_grad()
optimizer.step()
if e % kwargs['interval'] == 0:
checkpoint = {
'state_dict':
model.module.state_dict()
if args.cuda else model.state_dict(),
'epoch':
e,
'best_loss':
max(best_loss, loss.item()),
'optimizer':
optimizer.state_dict()
}
from DeepHyperX.batch_prune import MODEL_PRUNE_RESTORE_PATH
save_checkpoint(state=checkpoint,
filename=args.model + "_alpha" +
str(args.alpha) + "_pruned.pth",
dir=MODEL_PRUNE_RESTORE_PATH,
is_best=(loss.item() < best_loss))
if e % 30 == 0:
lr = optimizer.lr * 0.1
adjust_learning_rate(optimizer, lr, verbose=True)
epoch = 1
except KeyboardInterrupt:
"""important for a number of use cases. E.g., desired pruning percentage is never reached or the steps are too slow.
Excel file will only be generated if program runs through successfully"""
print("execution interrupted by user's ctrl+c\n")
end_next_turn_flag = True
validate(model,
val_loader,
topk,
cuda=args.cuda,
df_column_entry_dict=df_column_entry_dict)
def apply_band_selection(technique, dataset, predictions, mode, n_components,
df_column_entry_dict):
if df_column_entry_dict is None:
df_column_entry_dict = {
} # couldn't care less, this is a lazy way to make all accesses work
print("Dataset current shape: " + str(dataset.shape))
print_memory_metrics("before applying band selection method " + technique,
df_column_entry_dict)
from DeepHyperX.batch import PARAMETER_JSON
parameterFile = open(PARAMETER_JSON, "r")
import json
data = json.load(parameterFile)
parameterFile.close()
if technique in ["IncrementalPCA"]: # requires special method
dataset, _ = applyIncrementalPCA(dataset, n_components)
elif technique in data["image_compression"]["extraction"]["techniques"]:
extraction_object = None
if technique == "PCA":
from sklearn.decomposition import PCA
""" HybridSN: Exploring 3D-2D CNN Feature Hierarchy for Hyperspectral Image Classification
Source code used: https://github.com/gokriznastic/HybridSN/blob/master/Hybrid-Spectral-Net.ipynb
Paper: https://arxiv.org/abs/1902.06701
Good parameters: 30 components for Indian Pines, 15 for Salinas and Pavia University
"""
extraction_object = PCA(n_components=n_components, whiten=True)
elif technique == "KernelPCA":
from sklearn.decomposition import KernelPCA
extraction_object = KernelPCA(kernel="rbf",
n_components=n_components,
gamma=None,
fit_inverse_transform=True,
n_jobs=1)
elif technique == "SparsePCA":
"""Sparse PCA uses the links between the ACP and the SVD to extract the main components by solving a lower-order matrix approximation problem."""
from sklearn.decomposition import SparsePCA
extraction_object = SparsePCA(n_components=n_components,
alpha=0.0001,
n_jobs=-1)
elif technique == "LDA": # only supervised is supported, y is required
if mode != "supervised":
print(
"warning: mode other than supervised detected for lda, setting it to supervised...\n"
)
mode = "supervised"
# maximally n_classes - 1 columns, https://stackoverflow.com/questions/26963454/lda-ignoring-n-components
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
extraction_object = LinearDiscriminantAnalysis(
n_components=n_components)
elif technique == "SVD":
from sklearn.decomposition import TruncatedSVD
extraction_object = TruncatedSVD(n_components=n_components,
algorithm='randomized',
n_iter=5)
elif technique == "GRP":
from sklearn.random_projection import GaussianRandomProjection
extraction_object = GaussianRandomProjection(
n_components=n_components, eps=0.5)
elif technique == "SRP":
from sklearn.random_projection import SparseRandomProjection
extraction_object = SparseRandomProjection(
n_components=n_components,
density='auto',
eps=0.5,
dense_output=False)
elif technique == "MDS":
"""O(n^3), uses lots of memory for distance matrix (doesn't fit in 48GB), doesn't fit in GPU memory either, so basically unusable"""
from sklearn.manifold import MDS
extraction_object = MDS(n_components=n_components,
n_init=12,
max_iter=200,
metric=True,
n_jobs=16)
elif technique == "MiniBatch":
"""takes too long"""
from sklearn.decomposition import MiniBatchDictionaryLearning
extraction_object = MiniBatchDictionaryLearning(
n_components=n_components, batch_size=200, alpha=1, n_iter=1)
elif technique == "LLE":
# modified LLE requires n_neighbors >= n_components
"""execution takes 20 minutes or so, but it does work, just takes a long time"""
from sklearn.manifold import LocallyLinearEmbedding
extraction_object = LocallyLinearEmbedding(
n_components=n_components,
n_neighbors=100,
method='modified',
n_jobs=4)
elif technique == "ICA":
from sklearn.decomposition import FastICA
extraction_object = FastICA(n_components=n_components,
algorithm='parallel',
whiten=True,
max_iter=100)
elif technique == "FactorAnalysis":
from sklearn.decomposition import FactorAnalysis
extraction_object = FactorAnalysis(n_components=n_components) #75
elif technique == "ISOMAP":
from sklearn import manifold
extraction_object = manifold.Isomap(n_neighbors=5,
n_components=n_components,
n_jobs=-1)
elif technique == "t-SNE":
# like PCA, but non-linear (pca is linear)
from sklearn.manifold import TSNE
extraction_object = TSNE(n_components=n_components,
learning_rate=300,
perplexity=30,
early_exaggeration=12,
init='random')
elif technique == "UMAP":
# install umap-learn for this to work
import umap
extraction_object = umap.UMAP(n_neighbors=50,
min_dist=0.3,
n_components=n_components)
elif technique == "NMF":
# https://www.kaggle.com/remidi/dimensionality-reduction-techniques
from sklearn.decomposition import NMF
extraction_object = NMF(n_components=n_components,
init='nndsvdar',
random_state=420)
elif technique == "F*G":
# super fast and nice
from sklearn.cluster import FeatureAgglomeration
extraction_object = FeatureAgglomeration(n_clusters=n_components,
linkage='ward')
else:
raise ValueError("Unknown feature extraction technique: " +
technique)
start_mem_measurement()
start = time.time()
dataset, _ = applyFeatureExtraction(
dataset,
predictions,
extraction_object,
mode,
merged=(len(dataset.shape) == 4 and len(predictions.shape) == 3))
time_elapse = time.time() - start
event = 'applying band selection method (EXTRACTION) ' + technique
formatted_time = str(timedelta(seconds=time_elapse))
df_column_entry_dict['Time measurement at ' + event +
' [s]'] = time_elapse
print("\n" + event + " took " + formatted_time + " seconds\n")
event = "after applying band selection method " + technique
stop_mem_measurement(event, df_column_entry_dict)
print_memory_metrics(event, df_column_entry_dict)
elif technique in data["image_compression"]["selection"]["techniques"]:
selection_object = None
if technique == "RandomForest":
# Random forests or random decision forests are an ensemble learning method for classification, regression and other
# tasks that operates by constructing a multitude of decision trees at training time and outputting the class that is the mode of the classes (classification) or mean prediction (regression) of the individual trees.[1][2] Random decision forests correct for decision trees' habit of overfitting to their training set.[3]:587–588 https://en.wikipedia.org/wiki/Random_forest
from sklearn.ensemble import RandomForestClassifier
selection_object = RandomForestClassifier()
elif technique == "LogisticRegression":
from sklearn.linear_model import LogisticRegression
selection_object = LogisticRegression()
elif technique == "LinearRegression":
from sklearn.linear_model import LinearRegression
selection_object = LinearRegression()
elif technique == "LightGBM":
from lightgbm import LGBMClassifier
selection_object = LGBMClassifier()
else:
raise ValueError("Unknown feature selection technique: " +
technique)
start_mem_measurement()
start = time.time()
dataset, _ = applyFeatureSelection(
dataset,
predictions,
selection_object,
n_components,
mode,
merged=(len(dataset.shape) == 4 and len(predictions.shape) == 3))
time_elapse = time.time() - start
event = 'applying band selection method (SELECTION) ' + technique
formatted_time = str(timedelta(seconds=time_elapse))
df_column_entry_dict['Time measurement at ' + event +
' [s]'] = time_elapse
print("\n" + event + " took " + formatted_time + " seconds\n")
event = "after applying band selection method " + technique
stop_mem_measurement(event, df_column_entry_dict)
print_memory_metrics(event, df_column_entry_dict)
print("Dataset new shape: " + str(dataset.shape))
return dataset
def pb_inference(MODEL, path, quantize_afterwards=False):
import tensorflow as tf # Default graph is initialized when the library is imported
import os
from tensorflow.python.platform import gfile
from PIL import Image
import numpy as np
import scipy
from scipy import misc
import matplotlib.pyplot as plt
DATASET = "IndianPines"
print("Doing PB inference for model " + MODEL + "...")
# path = "D:/Experiments/winmltoolsQuantized/"
GRAPH_PB_PATH = path + MODEL + ".pb" #path to your .pb file
with tf.Graph().as_default() as graph: # Set default graph as graph
with tf.Session() as sess:
# Load the graph in graph_def
print("load graph")
# We load the protobuf file from the disk and parse it to retrive the unserialized graph_drf
with gfile.FastGFile(GRAPH_PB_PATH, 'rb') as f:
# Load IndianPines dataset
from DeepHyperX.datasets import get_dataset
img, gt, LABEL_VALUES, IGNORED_LABELS, RGB_BANDS, palette = get_dataset(
DATASET, "./DeepHyperX/Datasets/")
from DeepHyperX.utils import sample_gt
_, test_gt = sample_gt(gt, 0.8, mode='random')
hyperparams = {}
from DeepHyperX.utils import get_device
hyperparams.update({
'n_classes': 17,
'n_bands': 200,
'ignored_labels': IGNORED_LABELS,
'device': torch.device("cpu"), #get_device(0),
'dataset': "IndianPines"
})
hyperparams['supervision'] = 'full'
hyperparams['flip_augmentation'] = False
hyperparams['radiation_augmentation'] = False
hyperparams['mixture_augmentation'] = False
hyperparams['center_pixel'] = True
# model-specific params
if MODEL == "cao":
hyperparams['patch_size'] = 9 # patch_size
hyperparams['batch_size'] = 100
elif MODEL == "hu":
hyperparams['patch_size'] = 1 # patch_size
hyperparams['batch_size'] = 100
# output_tensor = 'mul_5:0'
elif MODEL == "he":
hyperparams['patch_size'] = 7 # patch_size
hyperparams['batch_size'] = 40
# output_tensor = 'add_17:0'#bs
# output_tensor = 'MatMul:0'#bs
# output_tensor = 'mul:0'#bs
elif MODEL == "santara":
hyperparams['patch_size'] = 3 # patch_size
hyperparams['batch_size'] = 200
# output_tensor = 'add_65:0'#bs
# output_tensor = 'LogSoftmax:0'
# output_tensor = 'MatMul_1:0'#bs
# output_tensor = 'transpose_124:0'
# output_tensor = 'mul_2:0'#bs
elif MODEL == "luo_cnn":
hyperparams['patch_size'] = 3 # patch_size
hyperparams['batch_size'] = 100
# output_tensor = 'add_5:0'
output_tensor = output_tensors[MODEL]
hyperparams['test_stride'] = 1 # default is 1
from DeepHyperX.datasets import HyperX
img_dataset = HyperX(data=img, gt=gt, hyperparams=hyperparams)
# Set FCN graph to the default graph
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
sess.graph.as_default()
# Import a graph_def into the current default Graph (In this case, the weights are (typically) embedded in the graph)
tf.import_graph_def(graph_def,
input_map=None,
return_elements=None,
name="",
op_dict=None,
producer_op_list=None)
# Print the name of operations in the session
# for op in graph.get_operations():
# print("Operation Name :", op.name) # Operation name
# print("Tensor Stats :", str(op.values())) # Tensor name
# INFERENCE Here
l_input = graph.get_tensor_by_name('0:0') # Input Tensor
l_output = graph.get_tensor_by_name(
output_tensor) # Output Tensor
# print("Shape of input : ", tf.shape(l_input))
# initialize_all_variables
tf.global_variables_initializer()
df_column_entry_dict = {}
print_memory_metrics("before PB Inference",
df_column_entry_dict)
start_mem_measurement()
start = time.time()
# get the right input data shape and run model
probs = test(img_dataset.data, hyperparams, sess, l_output,
l_input, MODEL)
time_elapse = time.time() - start
event = 'PB Inference'
formatted_time = str(timedelta(seconds=time_elapse))
df_column_entry_dict['Time measurement at ' + event +
' [s]'] = time_elapse
print("\n" + event + " took " + formatted_time + " seconds\n")
event = "after PB Inference"
stop_mem_measurement(event, df_column_entry_dict)
print_memory_metrics(event, df_column_entry_dict)
prediction = np.argmax(probs, axis=-1)
# goal: display accuracy metrics, incl. confusion matrix
from DeepHyperX.utils import metrics
run_results = metrics(
prediction,
test_gt,
ignored_labels=hyperparams['ignored_labels'],
n_classes=hyperparams['n_classes'])
mask = np.zeros(gt.shape, dtype='bool')
for l in IGNORED_LABELS:
mask[gt == l] = True
prediction[mask] = 0
results = []
results.append(run_results)
from DeepHyperX.utils import show_results
import visdom
viz = visdom.Visdom(env=MODEL + "_" + DATASET)
dataframe_grid = []
show_results(run_results,
viz,
label_values=LABEL_VALUES,
df_column_entry_dict=df_column_entry_dict)
dataframe_grid.append(list(df_column_entry_dict.values()))
import pandas as pd
frame = pd.DataFrame(dataframe_grid,
columns=list(df_column_entry_dict.keys()))
means = frame.mean()
frame = frame.append(means, ignore_index=True)
from DeepHyperX.batch import STORE_EXPERIMENT_LOCATION
frame.to_excel(path + MODEL + "_" + DATASET + ".xlsx",
index=False)
if quantize_afterwards:
print("Quantizing model " + MODEL + " after inference...\n")
img = tf.identity(tf.get_variable(
name="0",
dtype=tf.float32,
shape=tuple(expected_input_shapes[MODEL])),
name="0")
# img = tf.identity(tf.get_variable(name="Const_53", dtype=tf.float32, shape=tuple(expected_input_shapes[MODEL])), name="Const_53")
# img = tf.identity(tf.get_variable(name="foo", dtype=tf.float32, shape=tuple(expected_input_shapes[MODEL])), name="0")
out = tf.identity(tf.get_variable(
name=output_tensors[MODEL][:len(output_tensors[MODEL]) -
2],
dtype=tf.float32,
shape=tuple(expected_output_shapes[MODEL])),
name=output_tensors[MODEL]
[:len(output_tensors[MODEL]) -
2]) # cut out ":0" for valid tensor name
# out = tf.identity(tf.get_variable(name="bar", dtype=tf.float32, shape=tuple(expected_output_shapes[MODEL])), name=output_tensors[MODEL][:len(output_tensors[MODEL])-2]) # cut out ":0" for valid tensor name
sess.run(tf.global_variables_initializer())
converter = tf.lite.TFLiteConverter.from_session(
sess, [img], [out])
tflite_model = converter.convert()
open("converted_model.tflite", "wb").write(tflite_model)
"""
# Pass these data loaders around for pruning later on. Will be set in this model if-else
train_loader = None
val_loader = None
# Distinguish by model type first. If neural network, choose the appropriate one later
if MODEL == 'SVM_grid':
print("Running a grid search SVM")
# Grid search SVM (linear and RBF)
X_train, y_train = build_dataset(img, train_gt,
ignored_labels=IGNORED_LABELS)
class_weight = 'balanced' if CLASS_BALANCING else None
from sklearn.svm import SVC
clf = sklearn.svm.SVC(class_weight=class_weight)
from sklearn.model_selection import GridSearchCV
clf = sklearn.model_selection.GridSearchCV(clf, SVM_GRID_PARAMS, verbose=5, n_jobs=4)
print_memory_metrics("got model/before training", df_column_entry_dict)
start_mem_measurement()
start = time.time()
clf.fit(X_train, y_train)
time_elapse = time.time() - start
event = 'model.predict'
formatted_time = str(timedelta(seconds=time_elapse))
df_column_entry_dict['Time measurement at '+event+' [s]'] = time_elapse
print("\n"+event+" took "+ formatted_time + " seconds\n")
event = "after training"
stop_mem_measurement(event, df_column_entry_dict)