def calc_objective_value(weight, input_shape, x_train, y_train, x_test, y_test,
num_classes, epochs):
""" train and evaluate the network given specified weights
:param: - weight: weighting for the two objectives
- input_shape: training data input shapes, e.g. amount of pixel
- x_train: training data input (pictures)
- y_train: training data groundtruth (classification number)
- x_test: test data input (pictures)
- y_test: test data groundtruth (classification number)
- num_classes: amount of classes (e.g. 10 digits (0-9))
- epochs: how long the training will last
:return: - test_loss1: loss value on testing data (after training) from objective loss1 (e.g. Crossentropy)
- test_loss2: loss value on testing data (after training) from objective loss2 (e.g. L1 loss)
- L0ges: amount of nonzero weights in the dense layers of the model
- L1ges: L1 values of all weights in the dense layers of the model
- test_accuracy: accuracy value on the testing data
"""
### define and compile the model used for the training
weight_decay = 1e-4
model = lenet5multimodel(input_shape=input_shape,
weight_decay=weight_decay)
model.mcompile(optimizer=MAdam(multi=False,
learning_rate=learning_rate,
descent_weight1=weight[0],
descent_weight2=weight[1]),
loss1='sparse_categorical_crossentropy',
loss2=L1lossDense(model),
metrics=['accuracy'])
nonzero_weights1 = []
nonzero_weights2 = []
nonzero_weights3 = []
### perform a pruning step before the training starts
weights1 = model.get_layer('denselayer1').get_weights()
weights2 = model.get_layer('denselayer2').get_weights()
weights3 = model.get_layer(
'denselayer3').get_weights() # weights and biases of last
sparsified_weights1 = update_weights(weights1, 0.001)
sparsified_weights2 = update_weights(weights2, 0.001)
sparsified_weights3 = update_weights(weights3, 0.001)
model.get_layer('denselayer1').set_weights(sparsified_weights1)
model.get_layer('denselayer2').set_weights(sparsified_weights2)
model.get_layer('denselayer3').set_weights(sparsified_weights3)
nonzero_weights1.append(
[np.count_nonzero(model.get_layer('denselayer1').get_weights()[0])])
nonzero_weights2.append(
[np.count_nonzero(model.get_layer('denselayer2').get_weights()[0])])
nonzero_weights3.append(
[np.count_nonzero(model.get_layer('denselayer3').get_weights()[0])])
### define a callback function that performs pruning after each iteration (after each batch)
weight_callback_batch = LambdaCallback(on_batch_end=lambda batch, logs: [
model.get_layer(f"{name}").set_weights(
update_weights(model.get_layer(f"{name}").get_weights(), 0.001))
for name in ['denselayer1', 'denselayer2', 'denselayer3']
])
safe_nonzeroweights1 = LambdaCallback(on_epoch_end=lambda epoch, logs: [
nonzero_weights1.append([
np.count_nonzero(model.get_layer('denselayer1').get_weights()[0])
])
])
safe_nonzeroweights2 = LambdaCallback(on_epoch_end=lambda epoch, logs: [
nonzero_weights2.append([
np.count_nonzero(model.get_layer('denselayer2').get_weights()[0])
])
])
safe_nonzeroweights3 = LambdaCallback(on_epoch_end=lambda epoch, logs: [
nonzero_weights3.append([
np.count_nonzero(model.get_layer('denselayer3').get_weights()[0])
])
])
### start the training process
history = model.mfit(x_train,
y_train,
epochs=epochs,
validation_data=[x_test, y_test],
callbacks=[
weight_callback_batch, safe_nonzeroweights1,
safe_nonzeroweights2, safe_nonzeroweights3
])
### calculate some information about the weights in the trained model
weights1 = model.get_layer("denselayer1").get_weights()
L1w1 = sum(sum(sum(np.abs(weights1))))
L0w1 = np.count_nonzero(weights1[0]) + np.count_nonzero(weights1[1])
weights2 = model.get_layer("denselayer2").get_weights()
L1w2 = sum(sum(sum(np.abs(weights2))))
L0w2 = np.count_nonzero(weights2[0]) + np.count_nonzero(weights2[1])
weights3 = model.get_layer("denselayer3").get_weights()
L1w3 = sum(sum(sum(np.abs(weights3))))
L0w3 = np.count_nonzero(weights3[0]) + np.count_nonzero(weights3[1])
L0ges = L0w1 + L0w2 + L0w3
L1ges = L1w1 + L1w2 + L1w3
### evaluate the performance of the trained model on the test data
[test_loss1, test_loss2,
test_accuracy] = model.evaluate_multi(x_test, y_test)
return [
test_loss1, test_loss2, L0ges, L1ges, test_accuracy, nonzero_weights1,
nonzero_weights2, nonzero_weights3, history
]
seed(1)
set_random_seed(2)
# collect values while training (in evaluation mode)
loss1_values = []
loss2_values = []
loss_values = []
val_loss_values = []
accuracy_values = []
val_accuracy_values = []
nonzero_weights1 = []
nonzero_weights2 = []
nonzero_weights3 = []
if opt == 'multiadam':
lambdaconv = 1e-4
model = lenet5multimodel(input_shape=input_shape,
weight_decay=lambdaconv)
model.mcompile(optimizer=MAdam(multi=True,
split=False,
learning_rate=learning_rate),
loss1='sparse_categorical_crossentropy',
loss2=L1lossDense(model),
metrics=['accuracy'])
nonzero_weights1.append([
np.count_nonzero(
model.get_layer('denselayer1').get_weights()[0])
])
nonzero_weights2.append([
np.count_nonzero(
model.get_layer('denselayer2').get_weights()[0])
])
nonzero_weights3.append([