def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = get_model(x_train.shape[1], 10, y_train.shape[1])
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
new_config['class_name'] = new_config['class_name'].lower()
assert config == new_config
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = Sequential()
model.add(Dense(10, input_shape=(x_train.shape[1],)))
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
new_config['class_name'] = new_config['class_name'].lower()
assert config == new_config
# Test constraints.
model = Sequential()
dense = Dense(10,
input_shape=(x_train.shape[1],),
kernel_constraint=lambda x: 0. * x + 1.,
bias_constraint=lambda x: 0. * x + 2.,)
model.add(dense)
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.train_on_batch(x_train[:10], y_train[:10])
kernel, bias = dense.get_weights()
assert_allclose(kernel, 1.)
assert_allclose(bias, 2.)
def __init__(self,
model,
optimizer,
loss,
loss_weights,
metrics=["accuracy"],
features_col="features",
label_col="label",
batch_size=32,
num_epoch=1,
learning_rate=1.0):
assert isinstance(
optimizer,
(str, Optimizer
)), "'optimizer' must be a string or a Keras Optimizer instance"
assert isinstance(
features_col,
(str,
list)), "'features_col' must be a string or a list of strings"
assert isinstance(
label_col,
(str, list)), "'label_col' must be a string or a list of strings"
self.model = model
self.optimizer = {
'class_name': optimizer,
'config': {}
} if isinstance(optimizer, str) else serialize(optimizer)
self.loss = loss
self.loss_weights = loss_weights
self.metrics = metrics
self.features_column = [features_col] if isinstance(
features_col, str) else features_col
self.label_column = [label_col] if isinstance(label_col,
str) else label_col
self.batch_size = batch_size
self.num_epoch = num_epoch
self.max_mini_batches = 100
self.prefetching_thread = None
self.mini_batches = None
self.is_prefetching = True
self.worker_id = -1
self.learning_rate = learning_rate
self.num_inputs = len(self.features_column)
self.num_outputs = len(self.label_column)
self.current_epoch = 0
def testSerializationWithCustomOptimizer(self):
class MySGD(gradient_descent.SGD):
def __init__(self, *args, **kwargs):
super(MySGD, self).__init__(*args, **kwargs)
self.my_attribute = 123
opt = MySGD(2., momentum=0.5)
opt = loss_scale_optimizer.LossScaleOptimizer(opt,
initial_scale=2.,
dynamic_growth_steps=3.)
config = optimizers.serialize(opt)
custom_objects = {'MySGD': MySGD}
opt = optimizers.deserialize(config, custom_objects=custom_objects)
# Force hyperparameters to be created
opt.lr # pylint: disable=pointless-statement
self.evaluate(tf.compat.v1.global_variables_initializer())
self.assertEqual(self.evaluate(opt.lr), 2.)
self.assertEqual(self.evaluate(opt.inner_optimizer.momentum), 0.5)
self.assertEqual(self.evaluate(opt.loss_scale), 2.)
self.assertEqual(opt.dynamic_growth_steps, 3.)
self.assertEqual(opt.inner_optimizer.my_attribute, 123)
def testSerializationWithCustomOptimizer(self, opt_cls):
sgd_cls = type(create_sgd(opt_cls))
class MySGD(sgd_cls):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.my_attribute = 123
opt = MySGD(2., momentum=0.5)
opt = create_lso(opt, initial_scale=2., dynamic_growth_steps=3.)
config = optimizers.serialize(opt)
custom_objects = {'MySGD': MySGD}
opt = optimizers.deserialize(config, custom_objects=custom_objects)
# Force hyperparameters to be created
opt.learning_rate # pylint: disable=pointless-statement
self.evaluate(tf.compat.v1.global_variables_initializer())
self.assertEqual(self.evaluate(opt.learning_rate), 2.)
self.assertEqual(self._eval_if_tensor(opt.inner_optimizer.momentum), 0.5)
self.assertEqual(self.evaluate(opt.loss_scale), 2.)
self.assertEqual(opt.dynamic_growth_steps, 3.)
self.assertEqual(opt.inner_optimizer.my_attribute, 123)
def __init__(self, model, optimizer, loss, loss_weights, metrics=["accuracy"], features_col="features", label_col="label",
batch_size=32, num_epoch=1, learning_rate=1.0):
assert isinstance(optimizer, (str, Optimizer)), "'optimizer' must be a string or a Keras Optimizer instance"
assert isinstance(features_col, (str, list)), "'features_col' must be a string or a list of strings"
assert isinstance(label_col, (str, list)), "'label_col' must be a string or a list of strings"
self.model = model
self.optimizer = {'class_name': optimizer, 'config': {}} if isinstance(optimizer, str) else serialize(optimizer)
self.loss = loss
self.loss_weights = loss_weights
self.metrics= metrics
self.features_column = [features_col] if isinstance(features_col, str) else features_col
self.label_column = [label_col] if isinstance(label_col, str) else label_col
self.batch_size = batch_size
self.num_epoch = num_epoch
self.max_mini_batches = 100
self.prefetching_thread = None
self.mini_batches = None
self.is_prefetching = True
self.worker_id = -1
self.learning_rate = learning_rate
self.num_inputs = len(self.features_column)
self.num_outputs = len(self.label_column)
model.add(Dense(128))
model.add(Activation('relu'))
model.add(Dropout(0.2))
model.add(Dense(10))
model.add(Activation('softmax'))
# Create Spark context
conf = SparkConf().setAppName('Mnist_Spark_MLP').setMaster('local[8]')
sc = SparkContext(conf=conf)
# Build RDD from numpy features and labels
df = to_data_frame(sc, x_train, y_train, categorical=True)
test_df = to_data_frame(sc, x_test, y_test, categorical=True)
sgd = optimizers.SGD(lr=0.01, decay=1e-6, momentum=0.9, nesterov=True)
sgd_conf = optimizers.serialize(sgd)
# Initialize Spark ML Estimator
estimator = ElephasEstimator()
estimator.set_keras_model_config(model.to_yaml())
estimator.set_optimizer_config(sgd_conf)
estimator.set_mode("synchronous")
estimator.set_loss("categorical_crossentropy")
estimator.set_metrics(['acc'])
estimator.set_epochs(epochs)
estimator.set_batch_size(batch_size)
estimator.set_validation_split(0.1)
estimator.set_categorical_labels(True)
estimator.set_nb_classes(nb_classes)
# Fitting a model returns a Transformer
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = Sequential()
model.add(Dense(10, input_shape=(x_train.shape[1],)))
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
new_config['class_name'] = new_config['class_name'].lower()
assert config == new_config
# Test constraints.
model = Sequential()
dense = Dense(10,
input_shape=(x_train.shape[1],),
kernel_constraint=lambda x: 0. * x + 1.,
bias_constraint=lambda x: 0. * x + 2.,)
model.add(dense)
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.train_on_batch(x_train[:10], y_train[:10])
kernel, bias = dense.get_weights()
assert_allclose(kernel, 1.)
assert_allclose(bias, 2.)
# Test lr_mult
model = Sequential()
dense = (Dense(10, input_shape=(x_train.shape[1],), lr_mult=0.0))
model.add(dense)
model.add(Dense(10, lr_mult=0.0))
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
kernel, bias = dense.get_weights()
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
kernel_new, bias_new = dense.get_weights()
assert np.array_equal(kernel_new, kernel)
assert np.array_equal(bias_new, bias)
model = Sequential()
dense = (Dense(10, input_shape=(x_train.shape[1],), lr_mult=1.0))
model.add(dense)
dense2 = (Dense(10, lr_mult=0.0))
model.add(dense2)
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
kernel, bias = dense.get_weights()
kernel2, bias2 = dense2.get_weights()
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
kernel_new, bias_new = dense.get_weights()
kernel2_new, bias2_new = dense2.get_weights()
assert not np.array_equal(kernel_new, kernel)
assert not np.array_equal(bias_new, bias)
assert np.array_equal(kernel2_new, kernel2)
assert np.array_equal(bias2_new, bias2)
model.add(Activation('relu'))
model.add(Dropout(rate=0.3))
model.add(Dense(256, activity_regularizer=regularizers.l2(0.01)))
model.add(Activation('relu'))
model.add(Dropout(rate=0.3))
model.add(Dense(nb_classes))
model.add(Activation('sigmoid'))
model.compile(loss='binary_crossentropy', optimizer='adam')
# Model Summary
print(model.summary())
#Distributed Deep Learning
# Set and Serialize Optimizer
optimizer_conf = optimizers.Adam(lr=0.01)
opt_conf = optimizers.serialize(optimizer_conf)
# Initialize SparkML Estimator and Get Settings
estimator = ElephasEstimator()
estimator.setFeaturesCol("features")
estimator.setLabelCol("label_index")
estimator.set_keras_model_config(model.to_yaml())
estimator.set_categorical_labels(True)
estimator.set_nb_classes(nb_classes)
estimator.set_num_workers(1)
estimator.set_epochs(25)
estimator.set_batch_size(64)
estimator.set_verbosity(1)
estimator.set_validation_split(0.10)
estimator.set_optimizer_config(opt_conf)
estimator.set_mode("synchronous")
test_batchsize = 1
image_preprocessing = preprocess_image(normalize_lighting=normalize_lighting, min_value=min_value, max_value=max_value)
run_name = "run_" + datetime.datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
weights_dir = os.path.join("weights", run_name)
if not os.path.isdir(weights_dir):
print("Create directory " + weights_dir + " for saving weights")
os.makedirs(weights_dir)
results_dir = os.path.join("results", run_name)
if not os.path.isdir(results_dir):
print("Create directory " + results_dir + " for saving results")
os.makedirs(results_dir)
config = dict(height=height, width=width, channels=channels, normalize_lighting=normalize_lighting,
min_value=min_value, max_value=max_value, background_as_class=background_as_class,
augmentation_args=augmentation_args, unet_args=unet_args, use_custom_losses=use_custom_losses,
optimizer=optimizers.serialize(optimizer), seed=seed, data_dir=data_dir, epochs=epochs,
train_batchsize=train_batchsize)
with open(os.path.join(weights_dir, "config.json"), "w") as file:
json.dump(config, file, indent=4)
with open(os.path.join(results_dir, "config.json"), "w") as file:
json.dump(config, file, indent=4)
print("\nTrain dataset statistics:")
train_generator = ImageMaskGenerator(
train_dir,
augmentation_args=augmentation_args,
image_preprocessing=image_preprocessing,
mask_preprocessing=preprocess_mask,
background_as_class=background_as_class,
target_size=(height, width),
image_color_mode="rgb",
elapsed_training_Vectors = time.time() - start
elapsed_training_VectorsCpu = time.clock() - startCpu
elapsed_validation_Vectors = time.time() - start
elapsed_validation_VectorsCpu = time.clock() - startCpu
start = time.time()
startCpu = time.clock()
input_dim = train_df.select("features").first()[0].shape
logger.info(
f"We have {num_classes} classes and {input_dim[0]} features")
model = get_model(train_df, input_dim)
model.compile(optimizer=optimizer, loss=mywloss, metrics=['accuracy'])
adam = optimizers.nadam(lr=0.01)
opt_conf = optimizers.serialize(adam)
# Initialize SparkML Estimator and set all relevant properties
estimator = ElephasEstimator()
estimator.setFeaturesCol(
"features") # These two come directly from pyspark,
estimator.setLabelCol("target") # hence the camel case. Sorry :)
estimator.set_keras_model_config(
model.to_yaml()) # Provide serialized Keras model
estimator.set_categorical_labels(True)
estimator.set_nb_classes(num_classes)
estimator.set_num_workers(
10) # We just use one worker here. Feel free to adapt it.
estimator.set_epochs(2) # was max-epochs
estimator.set_batch_size(batch_size) # was 128
estimator.set_verbosity(2) # was 1