def testSparsityValueIsValid(self, schedule_type):
if schedule_type == 'constant_sparsity':
# pylint: disable=unnecessary-lambda
self._validate_sparsity(lambda s: pruning_schedule.ConstantSparsity(s, 0))
elif schedule_type == 'polynomial_decay':
self._validate_sparsity(
lambda s: pruning_schedule.PolynomialDecay(s, 0.8, 0, 10))
self._validate_sparsity(
lambda s: pruning_schedule.PolynomialDecay(0.2, s, 0, 10))
def testPruneStopAndRestartOnModel(self, save_restore_fn):
params = {
'pruning_schedule':
pruning_schedule.PolynomialDecay(0.2, 0.6, 0, 4, 3, 1),
'block_size': (1, 1),
'block_pooling_type':
'AVG'
}
model = prune.prune_low_magnitude(
keras_test_utils.build_simple_dense_model(), **params)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# Model hasn't been trained yet. Sparsity 0.0
test_utils.assert_model_sparsity(self, 0.0, model)
model.fit(np.random.rand(20, 10),
np_utils.to_categorical(np.random.randint(5, size=(20, 1)),
5),
batch_size=20,
callbacks=[pruning_callbacks.UpdatePruningStep()])
# Training has run only 1 step. Sparsity 0.2 (initial_sparsity)
test_utils.assert_model_sparsity(self, 0.2, model)
model = save_restore_fn(model)
model.fit(np.random.rand(20, 10),
np_utils.to_categorical(np.random.randint(5, size=(20, 1)),
5),
batch_size=20,
epochs=3,
callbacks=[pruning_callbacks.UpdatePruningStep()])
# Training has run all 4 steps. Sparsity 0.6 (final_sparsity)
test_utils.assert_model_sparsity(self, 0.6, model)
self._check_strip_pruning_matches_original(model, 0.6)
def testPruneStopAndRestart_PreservesSparsity(self, save_restore_fn):
# TODO(tfmot): renable once SavedModel preserves step again.
# This existed in TF 2.0 and 2.1 and should be reenabled in
# TF 2.3. b/151755698
if save_restore_fn.__name__ == '_save_restore_tf_model':
return
begin_step, end_step = 1, 4
params = self.params
params['pruning_schedule'] = pruning_schedule.PolynomialDecay(
0.2, 0.6, begin_step, end_step, 3, 1)
model = prune.prune_low_magnitude(
keras_test_utils.build_simple_dense_model(), **params)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# Model hasn't been trained yet. Sparsity 0.0
test_utils.assert_model_sparsity(self, 0.0, model)
# Train only 1 step. Sparsity 0.2 (initial_sparsity)
self._train_model(model, epochs=1)
test_utils.assert_model_sparsity(self, 0.2, model)
model = save_restore_fn(model)
# Training has run all 4 steps. Sparsity 0.6 (final_sparsity)
self._train_model(model, epochs=3)
test_utils.assert_model_sparsity(self, 0.6, model)
self._check_strip_pruning_matches_original(model, 0.6)
def testPruneWithPolynomialDecayPastEndStep_PreservesSparsity(
self, save_restore_fn):
begin_step, end_step = 0, 2
params = self.params
params['pruning_schedule'] = pruning_schedule.PolynomialDecay(
0.2, 0.6, begin_step, end_step, 3, 1)
model = prune.prune_low_magnitude(
keras_test_utils.build_simple_dense_model(), **params)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# Model hasn't been trained yet. Sparsity 0.0
test_utils.assert_model_sparsity(self, 0.0, model)
# Train 3 steps, past end_step. Sparsity 0.6 (final_sparsity)
self._train_model(model, epochs=3)
test_utils.assert_model_sparsity(self, 0.6, model)
model = save_restore_fn(model)
# Ensure sparsity is preserved.
test_utils.assert_model_sparsity(self, 0.6, model)
# Train one more step to ensure nothing happens that brings sparsity
# back below 0.6.
self._train_model(model, epochs=1)
test_utils.assert_model_sparsity(self, 0.6, model)
self._check_strip_pruning_matches_original(model, 0.6)
def testPruneStopAndRestart_PreservesSparsity(self, save_restore_fn):
begin_step, end_step = 0, 4
params = self.params
params['pruning_schedule'] = pruning_schedule.PolynomialDecay(
0.2, 0.6, begin_step, end_step, 3, 1)
model = prune.prune_low_magnitude(
keras_test_utils.build_simple_dense_model(), **params)
model.compile(loss='categorical_crossentropy',
optimizer='sgd',
metrics=['accuracy'])
# Model hasn't been trained yet. Sparsity 0.0
test_utils.assert_model_sparsity(self, 0.0, model)
# Train only 1 step. Sparsity 0.2 (initial_sparsity)
self._train_model(model, epochs=1)
test_utils.assert_model_sparsity(self, 0.2, model)
model = save_restore_fn(model)
# Training has run all 4 steps. Sparsity 0.6 (final_sparsity)
self._train_model(model, epochs=3)
test_utils.assert_model_sparsity(self, 0.6, model)
self._check_strip_pruning_matches_original(model, 0.6)
def _construct_pruning_schedule(
schedule_type, begin_step, end_step, frequency=10):
# Uses default values for sparsity. We're only testing begin_step, end_step
# and frequency here.
if schedule_type == 'constant_sparsity':
return pruning_schedule.ConstantSparsity(
0.5, begin_step, end_step, frequency)
elif schedule_type == 'polynomial_decay':
return pruning_schedule.PolynomialDecay(
0.2, 0.8, begin_step, end_step, 3, frequency)
def testSerializeDeserialize(self):
sparsity = pruning_schedule.PolynomialDecay(0.2, 0.6, 10, 20, 5, 10)
config = sparsity.get_config()
sparsity_deserialized = tf.keras.utils.deserialize_keras_object(
config,
custom_objects={
'ConstantSparsity': pruning_schedule.ConstantSparsity,
'PolynomialDecay': pruning_schedule.PolynomialDecay
})
self.assertEqual(sparsity.__dict__, sparsity_deserialized.__dict__)
def testPolynomialDecay_PrunesCorrectly(self):
sparsity = pruning_schedule.PolynomialDecay(0.2, 0.8, 100, 110, 3, 2)
step_100 = tf.Variable(100)
step_102 = tf.Variable(102)
step_105 = tf.Variable(105)
step_110 = tf.Variable(110)
compat.initialize_variables(self)
# These values were generated using tf.polynomial_decay with the same
# params in a colab to verify.
self.assertAllClose(0.2, self.evaluate(sparsity(step_100))[1])
self.assertAllClose(0.4928, self.evaluate(sparsity(step_102))[1])
self.assertAllClose(0.725, self.evaluate(sparsity(step_105))[1])
self.assertAllClose(0.8, self.evaluate(sparsity(step_110))[1])
def get_pruning_params(mode='prune',
initial_sparsity=0.0,
final_sparsity=0.8,
begin_step=2000,
end_step=4000,
frequency=200):
"""Gets pruning hyper-parameters."""
p_params = {}
if mode == 'prune':
p_params['pruning_schedule'] = pruning_schedule.PolynomialDecay(
initial_sparsity=initial_sparsity,
final_sparsity=final_sparsity,
begin_step=begin_step,
end_step=end_step,
frequency=frequency)
elif mode == 'constant':
p_params['pruning_schedule'] = pruning_schedule.ConstantSparsity(
target_sparsity=final_sparsity, begin_step=begin_step)
else:
raise ValueError('Mode: %s, is not valid' % mode)
return p_params
if __name__ == "__main__":
parameters = open("parameters.yml")
yamlparameters = yaml.load(parameters,Loader=yaml.FullLoader)
experiment = Experiment(api_key=yamlparameters["comet_api_key"],project_name='qkeras',auto_param_logging=True)
X_train, X_test, y_train, y_test = get_features(yamlparameters["DataDir"])
steps_per_epoch = int(len(X_train)/yamlparameters["Training_batch_size"])
pruning_params = {"pruning_schedule" : pruning_schedule.PolynomialDecay(initial_sparsity=0.0,
final_sparsity=yamlparameters["Sparsity"],
begin_step=yamlparameters["Pruning_begin_epoch"]*steps_per_epoch,
end_step=yamlparameters["Pruning_end_epoch"]*steps_per_epoch)}
#pruning_params = {"pruning_schedule" : pruning_schedule.ConstantSparsity(
# target_sparsity=yamlparameters["Sparsity"],
# begin_step=yamlparameters["Pruning_begin_epoch"]*steps_per_epoch,
# end_step=yamlparameters["Pruning_end_epoch"]*steps_per_epoch,
# frequency=yamlparameters["Pruning_frequency"]*steps_per_epoch)}
keras_model = models.qdense_model(Input(shape=X_train.shape[1:]),
l1Reg=yamlparameters["Training_regularization"],
bits=yamlparameters["Layer_bits"],
ints=yamlparameters["Layer_ints"])
keras_model = prune.prune_low_magnitude(keras_model, **pruning_params)
def _main():
annotation_path = 'model_data/combined.txt'
log_dir = 'logs/000/'
classes_path = 'model_data/voc_classes.txt'
anchors_path = 'model_data/yolo_anchors.txt'
class_names = get_classes(classes_path)
num_classes = len(class_names)
anchors = get_anchors(anchors_path)
model_path = 'model_data/'
init_model = model_path + '/pelee3'
new_pruned_keras_file = model_path + 'pruned_' + init_model
epochs = 100
init_epoch = 50
input_shape = (384, 286) # multiple of 32, hw
model = tf.keras.models.load_model(
init_model) # make sure you know what you freeze
logging = TensorBoard(log_dir=log_dir)
checkpoint = ModelCheckpoint(
log_dir + 'ep{epoch:03d}-loss{loss:.3f}-val_loss{val_loss:.3f}.h5',
monitor='val_loss',
save_weights_only=True,
save_best_only=True,
period=3)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.1,
patience=3,
verbose=1)
early_stopping = EarlyStopping(monitor='val_loss',
min_delta=0,
patience=10,
verbose=1)
val_split = 0.1
with open(annotation_path) as f:
lines = f.readlines()
np.random.seed(10101)
np.random.shuffle(lines)
np.random.seed(None)
num_val = int(len(lines) * val_split)
num_train = len(lines) - num_val
logdir = "d:/training"
# Train with frozen layers first, to get a stable loss.
# Adjust num epochs to your dataset. This step is enough to obtain a not bad model.
if False:
model.compile(
optimizer=SGD(lr=2e-2),
loss={
# use custom yolo_loss Lambda layer.
'yolo_loss': lambda y_true, y_pred: y_pred
})
batch_size = 16
print('Train on {} samples, val on {} samples, with batch size {}.'.
format(num_train, num_val, batch_size))
model.fit_generator(data_generator_wrapper(lines[:num_train],
batch_size, input_shape,
anchors, num_classes),
steps_per_epoch=max(1, num_train // batch_size),
validation_data=data_generator_wrapper(
lines[num_train:], batch_size, input_shape,
anchors, num_classes),
validation_steps=max(1, num_val // batch_size),
epochs=50,
initial_epoch=0,
callbacks=[logging, checkpoint])
model.save_weights(log_dir + 'trained_weights_stage_1.h5')
# Unfreeze and continue training, to fine-tune.
# Train longer if the result is not good.
if False:
for i in range(len(model.layers)):
model.layers[i].trainable = True
model.compile(optimizer=Adam(lr=1e-4),
loss={
'yolo_loss': lambda y_true, y_pred: y_pred
}) # recompile to apply the change
print('Unfreeze all of the layers.')
batch_size = 16 # note that more GPU memory is required after unfreezing the body
print('Train on {} samples, val on {} samples, with batch size {}.'.
format(num_train, num_val, batch_size))
model.fit_generator(data_generator_wrapper(lines[:num_train],
batch_size, input_shape,
anchors, num_classes),
steps_per_epoch=max(1, num_train // batch_size),
validation_data=data_generator_wrapper(
lines[num_train:], batch_size, input_shape,
anchors, num_classes),
validation_steps=max(1, num_val // batch_size),
epochs=epochs,
initial_epoch=init_epoch,
callbacks=[logging, checkpoint, reduce_lr])
model.save_weights(log_dir + 'trained_weights_final.h5')
# Further training if needed.
if True:
new_pruning_params = {
'pruning_schedule':
pruning_schedule.PolynomialDecay(initial_sparsity=0.30,
final_sparsity=0.60,
begin_step=0,
end_step=4,
frequency=100)
}
new_pruned_model = prune.prune_low_magnitude(model,
**new_pruning_params)
new_pruned_model.summary()
yolo_loss = YoloLoss(input_shape, num_classes, anchors, coord_scale,
class_scale, object_scale, no_object_scale)
new_pruned_model.compile(loss=yolo_loss,
optimizer='adam',
metrics=['accuracy'])
callbacks = [
sparsity.UpdatePruningStep(),
sparsity.PruningSummaries(log_dir=logdir, profile_batch=0)
]
model.fit_generator(data_generator_wrapper(lines[:num_train],
batch_size, input_shape,
anchors, num_classes),
steps_per_epoch=max(1, num_train // batch_size),
validation_data=data_generator_wrapper(
lines[num_train:], batch_size, input_shape,
anchors, num_classes),
validation_steps=max(1, num_val // batch_size),
epochs=4,
initial_epoch=init_epoch,
callbacks=[logging, checkpoint, reduce_lr])
score = new_pruned_model.evaluate(x_test, y_test, verbose=0)
print('Test loss:', score[0])
print('Test accuracy:', score[1])
final_model = sparsity.strip_pruning(pruned_model)
final_model.summary()
print('Saving pruned model to: ', new_pruned_keras_file)
tf.keras.models.save_model(final_model,
new_pruned_keras_file,
include_optimizer=False)
tflite_model_file = mode_path + "sparse_" + init_model
converter = tf.lite.TFLiteConverter.from_keras_model_file(final_model)
converter.optimizations = [tf.lite.Optimize.OPTIMIZE_FOR_SIZE]
tflite_model = converter.convert()
with open(tflite_model_file, 'wb') as f:
f.write(tflite_model)
x_test = keras.sequence.pad_sequences(x_test, maxlen=maxlen)
print("x_train shape:", x_train.shape)
print("x_test shape:", x_test.shape)
print("Build model...")
model = keras.models.Sequential()
model.add(keras.layers.Embedding(max_features, 128, input_length=maxlen))
model.add(keras.layers.LSTM(128)) # try using a GRU instead, for fun
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.Dense(1))
model.add(keras.layers.Activation("sigmoid"))
model = prune.prune_low_magnitude(
model,
pruning_schedule.PolynomialDecay(initial_sparsity=0.3,
final_sparsity=0.7,
begin_step=1000,
end_step=3000))
# try using different optimizers and different optimizer configs
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
print_model_sparsity(model)
print("Train...")
model.fit(x_train,
y_train,
batch_size=batch_size,
epochs=3,
callbacks=[pruning_callbacks.UpdatePruningStep()],
validation_data=(x_test, y_test))
def testRaisesErrorIfEndStepIsNegative(self):
with self.assertRaises(ValueError):
pruning_schedule.PolynomialDecay(0.4, 0.8, 10, -1)
GradientBoostingRegressor(n_estimators=15),
get_mlp_diabetes(25),
get_mlp_diabetes(50),
get_mlp_diabetes(100),
]
CONFIG_POOL_REG = [
Config(id=i, dataset_name=DATASET_NAME, classifier_model=clf)
for i, clf in enumerate(MODEL_POOL_REG)
]
prune_poly_50 = {
'pruning_schedule':
pruning_schedule.PolynomialDecay(initial_sparsity=0.00,
final_sparsity=0.50,
begin_step=50,
end_step=920,
frequency=50)
}
prune_poly_90 = {
'pruning_schedule':
pruning_schedule.PolynomialDecay(initial_sparsity=0.00,
final_sparsity=0.90,
begin_step=50,
end_step=920,
frequency=50)
}
prune_const_90 = {
'pruning_schedule':
opt = Optimizer(config,
api_key=yamlparameters["comet_api_key"],
project_name="NNqhmv6",
auto_metric_logging=True)
X_train, X_test, y_train, y_test = get_features(yamlparameters["DataDir"])
for experiment in opt.get_experiments():
steps_per_epoch = int(len(X_train) / yamlparameters["Training_batch_size"])
pruning_params = {
"pruning_schedule":
pruning_schedule.PolynomialDecay(
initial_sparsity=0.0,
final_sparsity=yamlparameters["Sparsity"],
begin_step=experiment.get_parameter("pruning_begin_epoch") *
steps_per_epoch,
end_step=experiment.get_parameter("pruning_end_epoch") *
steps_per_epoch)
}
keras_model = models.qdense_model(
Input(shape=X_train.shape[1:]),
l1Reg=experiment.get_parameter("Regularization"),
bits=yamlparameters["Layer_bits"],
ints=yamlparameters["Layer_ints"])
keras_model = prune.prune_low_magnitude(keras_model, **pruning_params)
startlearningrate = experiment.get_parameter("learning_rate")
adam = Adam(lr=startlearningrate,
beta_1=yamlparameters["Training_learning_beta1"],
print("Pad sequences (samples x time)")
x_train = sequence.pad_sequences(x_train, maxlen=maxlen)
x_test = sequence.pad_sequences(x_test, maxlen=maxlen)
print("x_train shape:", x_train.shape)
print("x_test shape:", x_test.shape)
print("Build model...")
model = keras.models.Sequential()
model.add(keras.layers.Embedding(max_features, 128, input_length=maxlen))
model.add(keras.layers.LSTM(128)) # try using a GRU instead, for fun
model.add(keras.layers.Dropout(0.5))
model.add(keras.layers.Dense(1))
model.add(keras.layers.Activation("sigmoid"))
model = prune.prune_low_magnitude(model, pruning_schedule.PolynomialDecay(
initial_sparsity=0.3, final_sparsity=0.7, begin_step=1000, end_step=3000))
# try using different optimizers and different optimizer configs
model.compile(loss="binary_crossentropy",
optimizer="adam",
metrics=["accuracy"])
print_model_sparsity(model)
print("Train...")
model.fit(x_train, y_train, batch_size=batch_size, epochs=3,
callbacks=[pruning_callbacks.UpdatePruningStep()],
validation_data=(x_test, y_test))
score, acc = model.evaluate(x_test, y_test,
batch_size=batch_size)
print_model_sparsity(model)
print("Test score:", score)
