def train(self, x_train, y_train, x_test, y_test):
if self.model is None:
print('Model has not been created yet, run createModel() first.')
return
else:
optimizer = None
if self.use_tf_privacy:
optimizer = DPKerasAdamOptimizer(
l2_norm_clip=self.params['l2_norm_clip'],
noise_multiplier=self.noise_multiplier,
learning_rate=self.params['learning_rate'],
num_microbatches=self.params['num_microbatches'])
else:
# optimizer=keras.optimizers.Adam(0.001)
optimizer = keras.optimizers.Adam(self.params['learning_rate'])
self.model.compile(
optimizer=optimizer,
loss=keras.losses.CategoricalCrossentropy(
from_logits=True
), # TODO: Check if from_logits=True is needed here
metrics=[keras.metrics.CategoricalAccuracy()])
self.model.fit(x_train,
y_train,
batch_size=self.params['batch_size'],
epochs=self.params['epochs'],
validation_split=self.params['validation_split'])
def train(self, x_train, y_train, x_test, y_test):
if self.model is None:
print('Model has not been created yet, run createModel() first.')
return
else:
optimizer = None
if self.use_tf_privacy:
optimizer = DPKerasAdamOptimizer(
l2_norm_clip=self.params['l2_norm_clip'],
noise_multiplier=self.noise_multiplier,
learning_rate=self.params['learning_rate'],
num_microbatches=self.params['num_microbatches'])
else:
# optimizer=keras.optimizers.Adam(0.001)
optimizer=keras.optimizers.Adam(self.params['learning_rate'])
self.model.compile(optimizer=optimizer,
loss=keras.losses.CategoricalCrossentropy(from_logits=True), # TODO: Check if from_logits=True is needed here
metrics=[keras.metrics.CategoricalAccuracy()])
# self.model.fit(x_train, y_train, batch_size=self.params['batch_size'], epochs=self.params['epochs'], validation_split=self.params['validation_split'])
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # dimesion reduction
rotation_range=0.1, # randomly rotate images in the range
zoom_range = 0.1, # Randomly zoom image
width_shift_range=0.1, # randomly shift images horizontally
height_shift_range=0.1, # randomly shift images vertically
horizontal_flip=False, # randomly flip images
vertical_flip=False) # randomly flip images
self.model.fit(datagen.flow(x_train, y_train), batch_size=self.params['batch_size'], epochs=self.params['epochs'])
def get_model():
input_img = tf.keras.Input(
shape=(width, height, 1), name="Input"
)
x = tf.keras.layers.Conv2D(
64, (3, 3), activation="relu", padding="same", name="Conv1_1"
)(input_img)
x = tf.keras.layers.BatchNormalization(name="bn1")(x)
x = tf.keras.layers.MaxPooling2D((2, 2), name="pool1")(x)
x = tf.keras.layers.Conv2D(
128, (3, 3), activation="relu", padding="same", name="Conv2_1"
)(x)
x = tf.keras.layers.BatchNormalization(name="bn4")(x)
x = tf.keras.layers.MaxPooling2D((2, 2), name="pool2")(x)
x = tf.keras.layers.Flatten(name="flatten")(x)
x = tf.keras.layers.Dense(
n_classes, activation="softmax", name="fc1"
)(x)
model = tf.keras.Model(inputs=input_img, outputs=x)
opt = DPKerasAdamOptimizer(
l2_norm_clip=diff_priv_config.max_grad_norm,
noise_multiplier=diff_priv_config.noise_multiplier,
num_microbatches=num_microbatches,
learning_rate=l_rate)
model.compile(
loss=tf.keras.losses.SparseCategoricalCrossentropy(
# need to calculare the loss per sample for the
# per sample / per microbatch gradient clipping
reduction=tf.losses.Reduction.NONE
),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
optimizer=opt)
return model
def get_model():
input_img = tf.keras.Input(
shape=(width, height, 1), name="Input"
)
x = tf.keras.layers.Conv2D(
64, (3, 3), activation="relu", padding="same", name="Conv1_1"
)(input_img)
x = tf.keras.layers.BatchNormalization(name="bn1")(x)
x = tf.keras.layers.MaxPooling2D((2, 2), name="pool1")(x)
x = tf.keras.layers.Conv2D(
128, (3, 3), activation="relu", padding="same", name="Conv2_1"
)(x)
x = tf.keras.layers.BatchNormalization(name="bn4")(x)
x = tf.keras.layers.MaxPooling2D((2, 2), name="pool2")(x)
x = tf.keras.layers.Flatten(name="flatten")(x)
x = tf.keras.layers.Dense(
n_classes, activation="softmax", name="fc1"
)(x)
model = tf.keras.Model(inputs=input_img, outputs=x)
opt = DPKerasAdamOptimizer(
l2_norm_clip=l2_norm_clip,
noise_multiplier=noise_multiplier,
num_microbatches=num_microbatches,
learning_rate=l_rate)
model.compile(
loss='sparse_categorical_crossentropy',
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
optimizer=opt)
return model
def fineTune(self, x_train, y_train):
for i in range(len(self.model.layers) - self.params['num_layers']):
self.model.layers[i].trainable = False
self.model.compile(optimizer=self.model.optimizer,
loss=self.model.loss,
metrics=[keras.metrics.CategoricalAccuracy()
]) # TODO: This line is probably needed
for i in range(len(self.model.layers)):
print(f'Layer: {i} | Trainable: {self.model.layers[i].trainable}')
self.model.summary()
optimizer = None
if self.use_tf_privacy:
optimizer = DPKerasAdamOptimizer(
l2_norm_clip=self.params['l2_norm_clip'],
noise_multiplier=self.noise_multiplier,
learning_rate=self.params['learning_rate'],
num_microbatches=self.params['num_microbatches'])
else:
# optimizer=keras.optimizers.Adam(0.001)
optimizer = keras.optimizers.Adam(self.params['learning_rate'])
self.model.compile(optimizer=optimizer,
loss=keras.losses.CategoricalCrossentropy(),
metrics=[keras.metrics.CategoricalAccuracy()])
# self.model.fit(x_train, y_train, batch_size=self.params['batch_size'], epochs=self.params['epochs'], validation_split=self.params['validation_split'])
datagen = ImageDataGenerator(
featurewise_center=False, # set input mean to 0 over the dataset
samplewise_center=False, # set each sample mean to 0
featurewise_std_normalization=
False, # divide inputs by std of the dataset
samplewise_std_normalization=False, # divide each input by its std
zca_whitening=False, # dimesion reduction
rotation_range=0.1, # randomly rotate images in the range
zoom_range=0.1, # Randomly zoom image
width_shift_range=0.1, # randomly shift images horizontally
height_shift_range=0.1, # randomly shift images vertically
horizontal_flip=False, # randomly flip images
vertical_flip=False) # randomly flip images
self.model.fit(datagen.flow(x_train, y_train),
batch_size=self.params['batch_size'],
epochs=self.params['epochs'])
weights_list_after = []
for layer in self.model.layers:
weights_list_after.append(layer.get_weights())
def prepare_learner(
data_loaders: Tuple[PrefetchDataset, PrefetchDataset, PrefetchDataset],
steps_per_epoch: int = 100,
vote_batches: int = 10,
learning_rate: float = 0.001,
diff_priv_config: Optional[DiffPrivConfig] = None,
num_microbatches: int = 4,
) -> KerasLearner:
"""
Creates new instance of KerasLearner
:param data_loaders: Tuple of train_loader and test_loader
:param steps_per_epoch: Number of batches per training epoch
:param vote_batches: Number of batches to get vote_accuracy
:param learning_rate: Learning rate for optimiser
:return: New instance of KerasLearner
"""
# 2D Convolutional model for image recognition
loss = "sparse_categorical_crossentropy"
optimizer = tf.keras.optimizers.Adam
input_img = tf.keras.Input(shape=(28, 28, 1), name="Input")
x = tf.keras.layers.Conv2D(32, (5, 5),
activation="relu",
padding="same",
name="Conv1_1")(input_img)
x = tf.keras.layers.MaxPooling2D((2, 2), name="pool1")(x)
x = tf.keras.layers.Conv2D(32, (5, 5),
activation="relu",
padding="same",
name="Conv2_1")(x)
x = tf.keras.layers.MaxPooling2D((2, 2), name="pool2")(x)
x = tf.keras.layers.Conv2D(64, (5, 5),
activation="relu",
padding="same",
name="Conv3_1")(x)
x = tf.keras.layers.MaxPooling2D((2, 2), name="pool3")(x)
x = tf.keras.layers.Flatten(name="flatten")(x)
x = tf.keras.layers.Dense(64, activation="relu", name="fc1")(x)
x = tf.keras.layers.Dense(10, activation="softmax", name="fc2")(x)
model = tf.keras.Model(inputs=input_img, outputs=x)
if diff_priv_config is not None:
opt = DPKerasAdamOptimizer(
l2_norm_clip=diff_priv_config.max_grad_norm,
noise_multiplier=diff_priv_config.noise_multiplier,
num_microbatches=num_microbatches,
learning_rate=learning_rate)
model.compile(
loss=tf.keras.losses.SparseCategoricalCrossentropy(
# need to calculare the loss per sample for the
# per sample / per microbatch gradient clipping
reduction=tf.losses.Reduction.NONE),
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
optimizer=opt)
else:
opt = optimizer(lr=learning_rate)
model.compile(loss=loss,
metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],
optimizer=opt)
learner = KerasLearner(
model=model,
train_loader=data_loaders[0],
vote_loader=data_loaders[1],
test_loader=data_loaders[2],
criterion="sparse_categorical_accuracy",
minimise_criterion=False,
model_fit_kwargs={"steps_per_epoch": steps_per_epoch},
model_evaluate_kwargs={"steps": vote_batches},
diff_priv_config=diff_priv_config,
)
return learner
args.network)
model = model_module.Network(**args_dict)
suffix = "{}.bs{}{}{}.ts{}{}.partition={}".format(
"" if not args.deep_supervision else ".dsup", args.batch_size,
".L1{}".format(args.l1) if args.l1 > 0 else "",
".L2{}".format(args.l2) if args.l2 > 0 else "", args.timestep,
".dp" if args.dp else "", args.partition)
model.final_name = args.prefix + model.say_name() + suffix
print("==> model.final_name:", model.final_name)
# Compile the model
print("==> compiling the model")
if args.dp:
optimizer = DPKerasAdamOptimizer(
l2_norm_clip=args.l2_norm_clip,
noise_multiplier=args.noise_multiplier,
num_microbatches=args.batch_size,
learning_rate=args.lr,
beta_1=args.beta_1)
else:
optimizer = tf.keras.optimizers.Adam(beta_1=args.beta_1,
learning_rate=args.lr)
if args.partition == 'none':
# other options are: 'mean_squared_error', 'mean_absolute_percentage_error'
loss_function = tf.keras.losses.MeanSquaredLogarithmicError(
reduction=tf.losses.Reduction.NONE if args.dp else tf.losses.
Reduction.AUTO)
else:
loss_function = tf.keras.losses.SparseCategoricalCrossentropy(
reduction=tf.losses.Reduction.NONE if args.dp else tf.losses.
Reduction.AUTO)