def from_config(cls, config, custom_objects=None):
config = config.copy() # Make a copy, since we mutate config
# If loss_scale is in config, we assume we are deserializing a
# LossScaleOptimizer from TF 2.3 or below. Otherwise, we assume we are
# deserializing a LossScaleOptimizer from TF 2.4 or above.
if 'loss_scale' in config:
config['loss_scale'] = keras_loss_scale_module.deserialize(
config['loss_scale'])
if (isinstance(config['loss_scale'], tf.mixed_precision.experimental.DynamicLossScale)
and config['loss_scale'].multiplier != 2):
raise ValueError('Cannot deserialize LossScaleOptimizer with a '
'DynamicLossScale whose multiplier is not 2. Got '
'DynamicLossScale: %s' % (config['loss_scale'],))
config['optimizer'] = optimizers.deserialize(
config['optimizer'], custom_objects=custom_objects)
return cls(**config)
# We convert the config, as generated by LossScaleOptimizer.get_config, to a
# version that can be passed to LossScaleOptimizerV1.__init__
if config['dynamic']:
config['loss_scale'] = tf.mixed_precision.experimental.DynamicLossScale(
config['initial_scale'], config['dynamic_growth_steps'], multiplier=2)
else:
config['loss_scale'] = tf.mixed_precision.experimental.FixedLossScale(
config['initial_scale'])
del config['dynamic']
del config['initial_scale']
del config['dynamic_growth_steps']
config['optimizer'] = optimizers.deserialize(
config.pop('inner_optimizer'), custom_objects=custom_objects)
return cls(**config)
def from_config(cls, config, custom_objects=None):
config = config.copy() # Make a copy, since we mutate config
if 'loss_scale' in config:
# If loss_scale is in config, we assume we are deserializing a
# LossScaleOptimizer from TF 2.3 or below. We convert the config so it
# can be deserialized in the current LossScaleOptimizer.
loss_scale = keras_loss_scale_module.deserialize(
config.pop('loss_scale'))
if isinstance(loss_scale,
tf.mixed_precision.experimental.FixedLossScale):
config['dynamic'] = False
config['initial_scale'] = loss_scale._loss_scale_value # pylint: disable=protected-access
elif isinstance(loss_scale,
tf.mixed_precision.experimental.DynamicLossScale):
config['dynamic'] = True
config['initial_scale'] = loss_scale.initial_loss_scale
config['dynamic_growth_steps'] = loss_scale.increment_period
if loss_scale.multiplier != 2:
raise ValueError(
'Cannot deserialize LossScaleOptimizer with a '
'DynamicLossScale whose multiplier is not 2. Got '
'DynamicLossScale: %s' % (loss_scale, ))
else:
raise ValueError(
'Serialized LossScaleOptimizers with a LossScale that is neither a '
'FixedLossScale nor a DynamicLossScale can no longer be '
'deserialized')
config['inner_optimizer'] = config.pop('optimizer')
config['inner_optimizer'] = optimizers.deserialize(
config['inner_optimizer'], custom_objects=custom_objects)
return cls(**config)
def testSerializationWithBuiltInOptimizer(self, use_v1):
opt = gradient_descent.SGD(2., momentum=0.5)
if use_v1:
loss_scale = tf.mixed_precision.experimental.DynamicLossScale(
initial_loss_scale=2., increment_period=3.)
opt = loss_scale_optimizer.LossScaleOptimizerV1(opt, loss_scale)
else:
opt = loss_scale_optimizer.LossScaleOptimizer(
opt, initial_scale=2., dynamic_growth_steps=3.)
config = optimizers.serialize(opt)
opt = optimizers.deserialize(config)
# Force hyperparameters to be created
opt.lr # pylint: disable=pointless-statement
self.evaluate(tf.compat.v1.global_variables_initializer())
self.assertEqual(self.evaluate(opt.lr), 2.)
self.assertEqual(self.evaluate(opt.inner_optimizer.momentum), 0.5)
self.assertEqual(self.evaluate(opt.loss_scale), 2.)
self.assertEqual(opt.dynamic_growth_steps, 3.)
self.assertTrue(opt.dynamic, 4.)
# Deserializing a LossScaleOptimizer always always results in a V2
# LossScaleOptimizer, even if serialized with a LossScaleOptimizerV1.
self.assertAllEqual(type(opt), loss_scale_optimizer.LossScaleOptimizer)
# Ensure the optimizer can be used
var = tf.Variable([5.0])
run_op = self._run_fn_with_grad_check(tf.distribute.get_strategy(),
var, opt, 2)()
self.evaluate(tf.compat.v1.global_variables_initializer())
self._run_if_in_graph_mode(run_op)
self.assertEqual(self.evaluate(var), [3.])
self.assertEqual(self.evaluate(opt.dynamic_counter), 1)
def build(self):
from keras.optimizers import deserialize
opt_config = {'class_name': self.name, 'config': self.config}
opt = deserialize(opt_config)
if self.horovod_wrapper:
import horovod.keras as hvd
if hasattr(opt, 'lr'):
opt.lr *= hvd.size()
opt = hvd.DistributedOptimizer(opt)
return opt
def clone_optimizer(optimizer):
if type(optimizer) is str:
return get(optimizer)
params = dict([(k, v) for k, v in optimizer.get_config().items()])
config = {
'class_name': optimizer.__class__.__name__,
'config': params,
}
clone = deserialize(config)
return clone
def _test_optimizer(optimizer, target=0.6):
x_train, y_train = get_test_data()
model = Sequential()
model.add(Dense(10, input_shape=(x_train.shape[1], )))
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=3, batch_size=16, verbose=0)
assert history.history['accuracy'][-1] >= target
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
new_config['class_name'] = new_config['class_name'].lower()
assert sorted(config.keys()) == sorted(new_config.keys())
# for k in config['config'].keys():
# assert config['config'][k] == new_config['config'][k]
# Test constraints.
model = Sequential()
dense = Dense(
10,
input_shape=(x_train.shape[1], ),
kernel_constraint=lambda x: 0. * x + 1.,
bias_constraint=lambda x: 0. * x + 2.,
)
model.add(dense)
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.train_on_batch(x_train[:10], y_train[:10])
kernel, bias = dense.get_weights()
assert_allclose(kernel, 1.)
assert_allclose(bias, 2.)
# Test saving.
model = Sequential()
model.add(Dense(1, input_dim=1))
model.compile(loss='mse', optimizer=optimizer)
model.fit(np.zeros((1, 1)), np.zeros((1, 1)))
_, fname = tempfile.mkstemp('.h5')
model.save(fname)
model2 = load_model(fname)
for w1, w2 in zip(model.get_weights(), model2.get_weights()):
assert_allclose(w1, w2)
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
# if the input optimizer is not a NormalizedOptimizer, wrap the optimizer
# with a default NormalizedOptimizer
if optimizer.__class__.__name__ != NormalizedOptimizer.__name__:
optimizer = NormalizedOptimizer(optimizer, normalization='l2')
model = Sequential()
model.add(Dense(10, input_shape=(x_train.shape[1], )))
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
# Test optimizer serialization and deserialization.
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
assert config == new_config
# Test weights saving and loading.
original_weights = optimizer.weights
model.save('temp.h5')
temp_model = load_model('temp.h5')
loaded_weights = temp_model.optimizer.weights
assert len(original_weights) == len(loaded_weights)
os.remove('temp.h5')
# Test constraints.
model = Sequential()
dense = Dense(
10,
input_shape=(x_train.shape[1], ),
kernel_constraint=lambda x: 0. * x + 1.,
bias_constraint=lambda x: 0. * x + 2.,
)
model.add(dense)
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.train_on_batch(x_train[:10], y_train[:10])
kernel, bias = dense.get_weights()
assert_allclose(kernel, 1.)
assert_allclose(bias, 2.)
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = get_model(x_train.shape[1], 10, y_train.shape[1])
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
assert config == new_config
def _load_optimizer(uncompiled_model, optimizer_metadata):
custom_objects = {}
def convert_custom_objects(obj):
"""Handles custom object lookup.
# Arguments
obj: object, dict, or list.
# Returns
The same structure, where occurrences
of a custom object name have been replaced
with the custom object.
"""
if isinstance(obj, list):
deserialized = []
for value in obj:
deserialized.append(convert_custom_objects(value))
return deserialized
if isinstance(obj, dict):
deserialized = {}
for key, value in obj.items():
deserialized[key] = convert_custom_objects(value)
return deserialized
if obj in custom_objects:
return custom_objects[obj]
return obj
# instantiate optimizer
training_config = optimizer_metadata.get('training_config')
if training_config is None:
warnings.warn('No training configuration found in save file: '
'the model was *not* compiled. '
'Compile it manually.')
return uncompiled_model
training_config = json.loads(training_config)
optimizer_config = training_config['optimizer_config']
optimizer = optimizers.deserialize(optimizer_config,
custom_objects=custom_objects)
# Recover loss functions and metrics.
loss = convert_custom_objects(training_config['loss'])
metrics = convert_custom_objects(training_config['metrics'])
sample_weight_mode = training_config['sample_weight_mode']
loss_weights = training_config['loss_weights']
# Compile model.
uncompiled_model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics,
loss_weights=loss_weights,
sample_weight_mode=sample_weight_mode)
model = uncompiled_model
return model
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = get_model(x_train.shape[1], 10, y_train.shape[1])
model.compile(loss="categorical_crossentropy",
optimizer=optimizer,
metrics=["accuracy"])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history["acc"][-1] >= target
config = optimizers.serialize(optimizer)
custom_objects = {optimizer.__class__.__name__: optimizer.__class__}
optim = optimizers.deserialize(config, custom_objects)
new_config = optimizers.serialize(optim)
assert config == new_config
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = get_model(x_train.shape[1], 10, y_train.shape[1])
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
new_config['class_name'] = new_config['class_name'].lower()
assert config == new_config
def load_bare_keras_optimizer(h5py_file, custom_objects=None):
if not custom_objects:
custom_objects = {}
def convert_custom_objects(obj):
"""Handles custom object lookup.
Arguments:
obj: object, dict, or list.
Returns:
The same structure, where occurrences
of a custom object name have been replaced
with the custom object.
"""
if isinstance(obj, list):
deserialized = []
for value in obj:
deserialized.append(convert_custom_objects(value))
return deserialized
if isinstance(obj, dict):
deserialized = {}
for key, value in obj.items():
deserialized[key] = convert_custom_objects(value)
return deserialized
if obj in custom_objects:
return custom_objects[obj]
return obj
optimizer, optimizer_weight_values = None, None
# instantiate optimizer
training_config = h5py_file.get('training_config')
training_config = json.loads(training_config[()].decode('utf-8'))
optimizer_config = training_config['optimizer_config']
optimizer = optimizers.deserialize(optimizer_config,
custom_objects=custom_objects)
if 'optimizer_weights' in h5py_file:
optimizer_weights_group = h5py_file['optimizer_weights']
optimizer_weight_names = [
n.decode('utf8')
for n in optimizer_weights_group.attrs['weight_names']
]
optimizer_weight_values = [
optimizer_weights_group[n].value for n in optimizer_weight_names
]
if optimizer_weight_values:
optimizer.set_weights(optimizer_weight_values)
return optimizer
def clone_optimizer(optimizer):
if type(optimizer) is str:
return optimizers.get(optimizer)
# Requires Keras 1.0.7 since get_config has breaking changes.
params = dict([(k, v) for k, v in optimizer.get_config().items()])
config = {
'class_name': optimizer.__class__.__name__,
'config': params,
}
if hasattr(optimizers, 'optimizer_from_config'):
# COMPATIBILITY: Keras < 2.0
clone = optimizers.optimizer_from_config(config)
else:
clone = optimizers.deserialize(config)
return clone
def clone_optimizer(optimizer):
if type(optimizer) is str:
return optimizers.get(optimizer)
# Requires Keras 1.0.7 since get_config has breaking changes.
params = dict([(k, v) for k, v in optimizer.get_config().items()])
config = {
'class_name': optimizer.__class__.__name__,
'config': params,
}
if hasattr(optimizers, 'optimizer_from_config'):
# COMPATIBILITY: Keras < 2.0
clone = optimizers.optimizer_from_config(config)
else:
clone = optimizers.deserialize(config)
return clone
def clone_optimizer(optimizer):
""" Helper function to clone a given optimizer
:param optimizer: prototype optimizer
:return:
"""
if type(optimizer) is str:
return optimizers.get(optimizer)
params = dict([(k, v) for k, v in optimizer.get_config().items()])
config = {
'class_name': optimizer.__class__.__name__,
'config': params,
}
new_optimizer = optimizers.deserialize(config)
return new_optimizer
def compile_args_from_training_config(training_config, custom_objects=None):
"""Return model.compile arguments from training config."""
if custom_objects is None:
custom_objects = {}
with generic_utils.CustomObjectScope(custom_objects):
optimizer_config = training_config["optimizer_config"]
optimizer = optimizers.deserialize(optimizer_config)
# Recover losses.
loss = None
loss_config = training_config.get("loss", None)
if loss_config is not None:
loss = _deserialize_nested_config(losses.deserialize, loss_config)
# Recover metrics.
metrics = None
metrics_config = training_config.get("metrics", None)
if metrics_config is not None:
metrics = _deserialize_nested_config(
_deserialize_metric, metrics_config
)
# Recover weighted metrics.
weighted_metrics = None
weighted_metrics_config = training_config.get("weighted_metrics", None)
if weighted_metrics_config is not None:
weighted_metrics = _deserialize_nested_config(
_deserialize_metric, weighted_metrics_config
)
sample_weight_mode = (
training_config["sample_weight_mode"]
if hasattr(training_config, "sample_weight_mode")
else None
)
loss_weights = training_config["loss_weights"]
return dict(
optimizer=optimizer,
loss=loss,
metrics=metrics,
weighted_metrics=weighted_metrics,
loss_weights=loss_weights,
sample_weight_mode=sample_weight_mode,
)
def prepare_model(self):
"""Prepares the model for training."""
# Set the Keras directory.
set_keras_base_directory()
if K.backend() == 'tensorflow':
# set GPU option allow_growth to False for GPU-enabled tensorflow
config = tf.ConfigProto()
config.gpu_options.allow_growth = False
sess = tf.Session(config=config)
K.set_session(sess)
# Deserialize the Keras model.
self.model = deserialize_keras_model(self.model)
self.optimizer = deserialize(self.optimizer)
# Compile the model with the specified loss and optimizer.
self.model.compile(loss=self.loss, loss_weights = self.loss_weights,
optimizer=self.optimizer, metrics=self.metrics)
def restore_model_state(model, checkpoint_path):
filepath = checkpoint_path + '_model_and_optimizer.h5'
f = h5py.File(filepath, mode='r')
load_weights_from_hdf5_group(f['model_weights'], model.layers)
training_config = f.attrs.get('training_config')
training_config = json.loads(training_config.decode('utf-8'))
optimizer_config = training_config['optimizer_config']
optimizer = optimizers.deserialize(optimizer_config)
# model.compile(optimizer=loaded_model.optimizer,
# loss=loaded_model.loss, # the loss function has no state
# metrics=loaded_model.metrics,
# loss_weights=loaded_model.loss_weights,
# sample_weight_mode=loaded_model.sample_weight_mode)
model.optimizer = optimizer
other_configs = np.load(checkpoint_path + '_other_logs.npz')
return other_configs['epoch'][0]
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = Sequential()
model.add(Dense(10, input_shape=(x_train.shape[1], )))
model.add(Activation("relu"))
model.add(Dense(y_train.shape[1]))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy",
optimizer=optimizer,
metrics=["accuracy"])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
accuracy = "acc" if get_backend() == "amd" else "accuracy"
assert history.history[accuracy][-1] >= target
config = k_optimizers.serialize(optimizer)
optim = k_optimizers.deserialize(config)
new_config = k_optimizers.serialize(optim)
config["class_name"] = config["class_name"].lower()
new_config["class_name"] = new_config["class_name"].lower()
assert config == new_config
# Test constraints.
if get_backend() == "amd":
# NB: PlaidML does not support constraints, so this test skipped for AMD backends
return
model = Sequential()
dense = Dense(
10,
input_shape=(x_train.shape[1], ),
kernel_constraint=lambda x: 0. * x + 1.,
bias_constraint=lambda x: 0. * x + 2.,
)
model.add(dense)
model.add(Activation("relu"))
model.add(Dense(y_train.shape[1]))
model.add(Activation("softmax"))
model.compile(loss="categorical_crossentropy",
optimizer=optimizer,
metrics=["accuracy"])
model.train_on_batch(x_train[:10], y_train[:10])
kernel, bias = dense.get_weights()
assert_allclose(kernel, 1.)
assert_allclose(bias, 2.)
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = Sequential()
model.add(Dense(10, input_shape=(x_train.shape[1], )))
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
# TODO PlaidML fails this test
assert history.history['acc'][-1] >= target
config = k_optimizers.serialize(optimizer)
optim = k_optimizers.deserialize(config)
new_config = k_optimizers.serialize(optim)
new_config['class_name'] = new_config['class_name'].lower()
assert config == new_config
# Test constraints.
model = Sequential()
dense = Dense(
10,
input_shape=(x_train.shape[1], ),
kernel_constraint=lambda x: 0. * x + 1.,
bias_constraint=lambda x: 0. * x + 2.,
)
model.add(dense)
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.train_on_batch(x_train[:10], y_train[:10])
kernel, bias = dense.get_weights()
assert_allclose(kernel, 1.)
assert_allclose(bias, 2.)
def testSerializationWithBuiltInOptimizer(self, lso_type):
if lso_type in ('v1', 'v2'):
opt = gradient_descent.SGD(2., momentum=0.5)
opt = loss_scale_optimizer.LossScaleOptimizer(
opt, initial_scale=2., dynamic_growth_steps=3.)
config = optimizers.serialize(opt)
if lso_type == 'v1':
# LossScaleOptimizerV1 was an older experimental version of LSO that is
# now deleted. The config had the same format as LSO but the class
# name was different. This tests that LSO V1 configs can still be
# deserialized, which are deserialized as a (non-V1) LSO
config['class_name'] = 'LossScaleOptimizerV1'
else:
opt = sgd_experimental.SGD(2., momentum=0.5)
opt = loss_scale_optimizer.LossScaleOptimizerV3(
opt, initial_scale=2., dynamic_growth_steps=3)
config = optimizers.serialize(opt)
opt = optimizers.deserialize(config)
# Force hyperparameters to be created
opt.learning_rate # pylint: disable=pointless-statement
self.evaluate(tf.compat.v1.global_variables_initializer())
self.assertEqual(self.evaluate(opt.learning_rate), 2.)
self.assertEqual(self._eval_if_tensor(opt.inner_optimizer.momentum), 0.5)
self.assertEqual(self.evaluate(opt.loss_scale), 2.)
self.assertEqual(opt.dynamic_growth_steps, 3.)
self.assertTrue(opt.dynamic)
if lso_type in ('v1', 'v2'):
self.assertEqual(type(opt), loss_scale_optimizer.LossScaleOptimizer)
else:
self.assertEqual(type(opt), loss_scale_optimizer.LossScaleOptimizerV3)
# Ensure the optimizer can be used
var = tf.Variable([5.0])
run_op = self._run_fn_with_grad_check(
tf.distribute.get_strategy(), var, opt, 2)()
self.evaluate(tf.compat.v1.global_variables_initializer())
self._run_if_in_graph_mode(run_op)
self.assertEqual(self.evaluate(var), [3.])
self.assertEqual(self.evaluate(opt.dynamic_counter), 1)
def testSerializationWithCustomOptimizer(self):
class MySGD(gradient_descent.SGD):
def __init__(self, *args, **kwargs):
super(MySGD, self).__init__(*args, **kwargs)
self.my_attribute = 123
opt = MySGD(2., momentum=0.5)
opt = loss_scale_optimizer.LossScaleOptimizer(opt,
initial_scale=2.,
dynamic_growth_steps=3.)
config = optimizers.serialize(opt)
custom_objects = {'MySGD': MySGD}
opt = optimizers.deserialize(config, custom_objects=custom_objects)
# Force hyperparameters to be created
opt.lr # pylint: disable=pointless-statement
self.evaluate(tf.compat.v1.global_variables_initializer())
self.assertEqual(self.evaluate(opt.lr), 2.)
self.assertEqual(self.evaluate(opt.inner_optimizer.momentum), 0.5)
self.assertEqual(self.evaluate(opt.loss_scale), 2.)
self.assertEqual(opt.dynamic_growth_steps, 3.)
self.assertEqual(opt.inner_optimizer.my_attribute, 123)
def testSerializationWithCustomOptimizer(self, opt_cls):
sgd_cls = type(create_sgd(opt_cls))
class MySGD(sgd_cls):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.my_attribute = 123
opt = MySGD(2., momentum=0.5)
opt = create_lso(opt, initial_scale=2., dynamic_growth_steps=3.)
config = optimizers.serialize(opt)
custom_objects = {'MySGD': MySGD}
opt = optimizers.deserialize(config, custom_objects=custom_objects)
# Force hyperparameters to be created
opt.learning_rate # pylint: disable=pointless-statement
self.evaluate(tf.compat.v1.global_variables_initializer())
self.assertEqual(self.evaluate(opt.learning_rate), 2.)
self.assertEqual(self._eval_if_tensor(opt.inner_optimizer.momentum), 0.5)
self.assertEqual(self.evaluate(opt.loss_scale), 2.)
self.assertEqual(opt.dynamic_growth_steps, 3.)
self.assertEqual(opt.inner_optimizer.my_attribute, 123)
def load_optimizer(model, hdf_file):
# instantiate optimizer
training_config = hdf_file.attrs.get('training_config')
training_config = json.loads(training_config.decode('utf-8'))
optimizer_config = training_config['optimizer_config']
optimizer = optimizers.deserialize(optimizer_config, custom_objects={})
# Recover loss functions and metrics.
loss = convert_custom_objects(training_config['loss'])
metrics = convert_custom_objects(training_config['metrics'])
sample_weight_mode = training_config['sample_weight_mode']
loss_weights = training_config['loss_weights']
# Compile model.
model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics,
loss_weights=loss_weights,
sample_weight_mode=sample_weight_mode)
# Set optimizer weights.
if 'optimizer_weights' in hdf_file:
# Build train function (to get weight updates).
if isinstance(model, Sequential):
model.model._make_train_function()
else:
model._make_train_function()
optimizer_weights_group = hdf_file['optimizer_weights']
optimizer_weight_names = [
n.decode('utf8')
for n in optimizer_weights_group.attrs['weight_names']
]
optimizer_weight_values = [
optimizer_weights_group[n] for n in optimizer_weight_names
]
model.optimizer.set_weights(optimizer_weight_values)
def _test_optimizer(optimizer, target=0.75):
x_train, y_train = get_test_data()
model = Sequential()
model.add(Dense(10, input_shape=(x_train.shape[1],)))
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
history = model.fit(x_train, y_train, epochs=2, batch_size=16, verbose=0)
assert history.history['acc'][-1] >= target
config = optimizers.serialize(optimizer)
optim = optimizers.deserialize(config)
new_config = optimizers.serialize(optim)
new_config['class_name'] = new_config['class_name'].lower()
assert config == new_config
# Test constraints.
model = Sequential()
dense = Dense(10,
input_shape=(x_train.shape[1],),
kernel_constraint=lambda x: 0. * x + 1.,
bias_constraint=lambda x: 0. * x + 2.,)
model.add(dense)
model.add(Activation('relu'))
model.add(Dense(y_train.shape[1]))
model.add(Activation('softmax'))
model.compile(loss='categorical_crossentropy',
optimizer=optimizer,
metrics=['accuracy'])
model.train_on_batch(x_train[:10], y_train[:10])
kernel, bias = dense.get_weights()
assert_allclose(kernel, 1.)
assert_allclose(bias, 2.)
def run(params):
args = candle.ArgumentStruct(**params)
seed = args.rng_seed
candle.set_seed(seed)
# # Construct extension to save model
# ext = p1b1.extension_from_parameters(params, '.keras')
# candle.verify_path(params['save_path'])
# prefix = '{}{}'.format(params['save_path'], ext)
# logfile = params['logfile'] if params['logfile'] else prefix+'.log'
# candle.set_up_logger(logfile, p1b1.logger, params['verbose'])
#p1b1.logger.info('Params: {}'.format(params))
# Get default parameters for initialization and optimizer functions
keras_defaults = candle.keras_default_config()
# Load dataset
x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels = p1b1.load_data(
params, seed)
# cache_file = 'data_l1000_cache.h5'
# save_cache(cache_file, x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels)
# x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels = load_cache(cache_file)
# p1b1.logger.info("Shape x_train: {}".format(x_train.shape))
# p1b1.logger.info("Shape x_val: {}".format(x_val.shape))
#p1b1.logger.info("Shape x_test: {}".format(x_test.shape))
# p1b1.logger.info("Range x_train: [{:.3g}, {:.3g}]".format(np.min(x_train), np.max(x_train)))
# p1b1.logger.info("Range x_val: [{:.3g}, {:.3g}]".format(np.min(x_val), np.max(x_val)))
#p1b1.logger.info("Range x_test: [{:.3g}, {:.3g}]".format(np.min(x_test), np.max(x_test)))
# p1b1.logger.debug('Class labels')
# for i, label in enumerate(y_labels):
# p1b1.logger.debug(' {}: {}'.format(i, label))
# clf = build_type_classifier(x_train, y_train, x_val, y_val)
n_classes = len(y_labels)
cond_train = y_train
cond_val = y_val
cond_test = y_test
input_dim = x_train.shape[1]
cond_dim = cond_train.shape[1]
latent_dim = params['latent_dim']
activation = params['activation']
dropout = params['dropout']
dense_layers = params['dense']
dropout_layer = AlphaDropout if params['alpha_dropout'] else Dropout
# Initialize weights and learning rule
initializer_weights = candle.build_initializer(params['initialization'],
keras_defaults, seed)
initializer_bias = candle.build_initializer('constant', keras_defaults, 0.)
if dense_layers is not None:
if type(dense_layers) != list:
dense_layers = list(dense_layers)
else:
dense_layers = []
# Encoder Part
x_input = Input(shape=(input_dim, ))
cond_input = Input(shape=(cond_dim, ))
h = x_input
if params['model'] == 'cvae':
h = keras.layers.concatenate([x_input, cond_input])
for i, layer in enumerate(dense_layers):
if layer > 0:
x = h
h = Dense(layer,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias)(h)
if params['residual']:
try:
h = keras.layers.add([h, x])
except ValueError:
pass
if params['batch_normalization']:
h = BatchNormalization()(h)
if dropout > 0:
h = dropout_layer(dropout)(h)
if params['model'] == 'ae':
encoded = Dense(latent_dim,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias)(h)
else:
epsilon_std = params['epsilon_std']
z_mean = Dense(latent_dim, name='z_mean')(h)
z_log_var = Dense(latent_dim, name='z_log_var')(h)
encoded = z_mean
def vae_loss(x, x_decoded_mean):
xent_loss = binary_crossentropy(x, x_decoded_mean)
kl_loss = -0.5 * K.sum(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return K.mean(xent_loss + kl_loss / input_dim)
def sampling(params):
z_mean_, z_log_var_ = params
batch_size = K.shape(z_mean_)[0]
epsilon = K.random_normal(shape=(batch_size, latent_dim),
mean=0.,
stddev=epsilon_std)
return z_mean_ + K.exp(z_log_var_ / 2) * epsilon
z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var])
if params['model'] == 'cvae':
z_cond = keras.layers.concatenate([z, cond_input])
# Build autoencoder model
if params['model'] == 'cvae':
# encoder = Model([x_input, cond_input], encoded)
# decoder = Model([decoder_input, cond_input], decoded)
# model = Model([x_input, cond_input], decoder([z, cond_input]))
loss = vae_loss
metrics = [xent, corr, mse]
elif params['model'] == 'vae':
# encoder = Model(x_input, encoded)
# decoder = Model(decoder_input, decoded)
# model = Model(x_input, decoder(z))
loss = vae_loss
metrics = [xent, corr, mse]
else:
# encoder = Model(x_input, encoded)
# decoder = Model(decoder_input, decoded)
# model = Model(x_input, decoder(encoded))
loss = params['loss']
metrics = [xent, corr]
# Define optimizer
# optimizer = candle.build_optimizer(params['optimizer'],
# params['learning_rate'],
# keras_defaults)
optimizer = optimizers.deserialize({
'class_name': params['optimizer'],
'config': {}
})
base_lr = params['base_lr'] or K.get_value(optimizer.lr)
if params['learning_rate']:
K.set_value(optimizer.lr, params['learning_rate'])
if params['model'] == 'cvae':
# inputs = [x_train, cond_train]
# val_inputs = [x_val, cond_val]
test_inputs = [x_test, cond_test]
else:
# inputs = x_train
# val_inputs = x_val
test_inputs = x_test
test_outputs = x_test
model_name = params['model_name']
# load json and create model
json_file = open('{}.{}.model.json'.format(model_name, params['model']),
'r')
loaded_model_json = json_file.read()
json_file.close()
loaded_model_json = model_from_json(loaded_model_json)
# load weights into new model
loaded_model_json.load_weights('{}.{}.weights.h5'.format(
model_name, params['model']))
print("Loaded model from disk")
# evaluate loaded model on test data
loaded_model_json.compile(loss=loss, optimizer=optimizer, metrics=metrics)
x_pred = loaded_model_json.predict(test_inputs)
scores = p1b1.evaluate_autoencoder(x_pred, x_test)
# p1b1.logger.info('\nEvaluation on test data: {}'.format(scores))
print('Evaluation on test data: {}'.format(scores))
# load encoder
encoder = load_model('{}.{}.encoder.h5'.format(model_name,
params['model']))
print("Loaded encoder from disk")
x_test_encoded = encoder.predict(test_inputs,
batch_size=params['batch_size'])
y_test_classes = np.argmax(y_test, axis=1)
candle.plot_scatter(x_test_encoded, y_test_classes,
'{}.{}.latent'.format(model_name, params['model']))
if params['tsne']:
tsne = TSNE(n_components=2, random_state=seed)
x_test_encoded_tsne = tsne.fit_transform(x_test_encoded)
candle.plot_scatter(
x_test_encoded_tsne, y_test_classes,
'{}.{}.latent.tsne'.format(model_name, params['model']))
def train(config_path: str):
config_file = open(config_path)
configs = json.load(config_file)
model_config = configs["model"]
if model_config["architecture"] == "UNet":
model = UNet(batch_normalize=model_config["batch_normalize"])
elif model_config["architecture"] == "UNetSS":
model = UNetSS(batch_normalize=model_config["batch_normalize"])
else:
raise ValueError("unknown segmentation architecture")
optimizer_config = dict()
optimizer_config['class_name'] = configs["optimizer"]["name"]
optimizer_config['config'] = configs["optimizer"]["args"]
optimizer = optimizers.deserialize(optimizer_config)
model.compile(optimizer=optimizer,
loss='categorical_crossentropy',
metrics=['accuracy'])
train_root_dir = Path(configs["train_data"]["root_dir"],
settings.train_subdir_name)
validation_root_dir = Path(configs["train_data"]["root_dir"],
settings.validation_subdir_name)
batch_size = configs["train_data"]["batch_size"]
label = SizeLabel if configs["train_data"]["use_size_label"] else BinLabel
train_output_branched = configs["train_data"]["branched"]
train_data_generator = data_generator(train_root_dir, batch_size, label,
train_output_branched)
validation_data_generator = data_generator(validation_root_dir, batch_size,
label, train_output_branched)
train_sample_n = \
len(list(train_root_dir.joinpath(settings.image_subdir_name, settings.dummycls_name).glob('./*.png')))
validation_sample_n = \
len(list(validation_root_dir.joinpath(settings.image_subdir_name, settings.dummycls_name).glob('./*.png')))
epochs = configs["train_data"]["epochs"]
output_dir = Path(configs["output_dir"])
if not output_dir.exists():
output_dir.mkdir(parents=True)
callbacks = [
CSVLogger(output_dir.joinpath('training.csv'), append=True),
ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001),
ModelCheckpoint(output_dir.joinpath('weights_{epoch}.h5').as_posix(),
monitor='val_loss',
save_best_only=True,
save_weights_only=True,
mode='auto',
period=5)
]
model.fit_generator(train_data_generator,
int(math.ceil(train_sample_n / batch_size)),
callbacks=callbacks,
validation_data=validation_data_generator,
validation_steps=validation_sample_n,
epochs=epochs)
model.save_weights(
output_dir.joinpath('weights_{}.h5'.format(epochs)).as_posix())
def run(params):
args = Struct(**params)
set_seed(args.rng_seed)
ext = extension_from_parameters(args)
verify_path(args.save_path)
prefix = args.save_path + ext
logfile = args.logfile if args.logfile else prefix + '.log'
set_up_logger(logfile, args.verbose)
logger.info('Params: {}'.format(params))
if (len(args.gpus) > 0):
import tensorflow as tf
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
config.gpu_options.visible_device_list = ",".join(map(str, args.gpus))
K.set_session(tf.Session(config=config))
loader = CombinedDataLoader(seed=args.rng_seed)
loader.load(
cache=args.cache,
ncols=args.feature_subsample,
agg_dose=args.agg_dose,
cell_features=args.cell_features,
drug_features=args.drug_features,
drug_median_response_min=args.drug_median_response_min,
drug_median_response_max=args.drug_median_response_max,
use_landmark_genes=args.use_landmark_genes,
use_filtered_genes=args.use_filtered_genes,
cell_feature_subset_path=args.cell_feature_subset_path
or args.feature_subset_path,
drug_feature_subset_path=args.drug_feature_subset_path
or args.feature_subset_path,
preprocess_rnaseq=args.preprocess_rnaseq,
single=args.single,
train_sources=args.train_sources,
test_sources=args.test_sources,
embed_feature_source=not args.no_feature_source,
encode_response_source=not args.no_response_source,
)
target = args.agg_dose or 'Growth'
val_split = args.validation_split
train_split = 1 - val_split
if args.export_csv:
fname = args.export_csv
loader.partition_data(cv_folds=args.cv,
train_split=train_split,
val_split=val_split,
cell_types=args.cell_types,
by_cell=args.by_cell,
by_drug=args.by_drug,
cell_subset_path=args.cell_subset_path,
drug_subset_path=args.drug_subset_path)
train_gen = CombinedDataGenerator(loader,
batch_size=args.batch_size,
shuffle=args.shuffle)
val_gen = CombinedDataGenerator(loader,
partition='val',
batch_size=args.batch_size,
shuffle=args.shuffle)
x_train_list, y_train = train_gen.get_slice(size=train_gen.size,
dataframe=True,
single=args.single)
x_val_list, y_val = val_gen.get_slice(size=val_gen.size,
dataframe=True,
single=args.single)
df_train = pd.concat([y_train] + x_train_list, axis=1)
df_val = pd.concat([y_val] + x_val_list, axis=1)
df = pd.concat([df_train, df_val]).reset_index(drop=True)
if args.growth_bins > 1:
df = uno_data.discretize(df, 'Growth', bins=args.growth_bins)
df.to_csv(fname, sep='\t', index=False, float_format="%.3g")
return
if args.export_data:
fname = args.export_data
loader.partition_data(cv_folds=args.cv,
train_split=train_split,
val_split=val_split,
cell_types=args.cell_types,
by_cell=args.by_cell,
by_drug=args.by_drug,
cell_subset_path=args.cell_subset_path,
drug_subset_path=args.drug_subset_path)
train_gen = CombinedDataGenerator(loader,
batch_size=args.batch_size,
shuffle=args.shuffle)
val_gen = CombinedDataGenerator(loader,
partition='val',
batch_size=args.batch_size,
shuffle=args.shuffle)
store = pd.HDFStore(fname, complevel=9, complib='blosc:snappy')
config_min_itemsize = {'Sample': 30, 'Drug1': 10}
if not args.single:
config_min_itemsize['Drug2'] = 10
for partition in ['train', 'val']:
gen = train_gen if partition == 'train' else val_gen
for i in range(gen.steps):
x_list, y = gen.get_slice(size=args.batch_size,
dataframe=True,
single=args.single)
for j, input_feature in enumerate(x_list):
input_feature.columns = [''] * len(input_feature.columns)
store.append('x_{}_{}'.format(partition, j),
input_feature.astype('float32'),
format='table',
data_column=True)
store.append('y_{}'.format(partition),
y.astype({target: 'float32'}),
format='table',
data_column=True,
min_itemsize=config_min_itemsize)
logger.info('Generating {} dataset. {} / {}'.format(
partition, i, gen.steps))
store.close()
logger.info('Completed generating {}'.format(fname))
return
loader.partition_data(cv_folds=args.cv,
train_split=train_split,
val_split=val_split,
cell_types=args.cell_types,
by_cell=args.by_cell,
by_drug=args.by_drug,
cell_subset_path=args.cell_subset_path,
drug_subset_path=args.drug_subset_path)
model = build_model(loader, args)
logger.info('Combined model:')
model.summary(print_fn=logger.info)
# plot_model(model, to_file=prefix+'.model.png', show_shapes=True)
if args.cp:
model_json = model.to_json()
with open(prefix + '.model.json', 'w') as f:
print(model_json, file=f)
def warmup_scheduler(epoch):
lr = args.learning_rate or base_lr * args.batch_size / 100
if epoch <= 5:
K.set_value(model.optimizer.lr,
(base_lr * (5 - epoch) + lr * epoch) / 5)
logger.debug('Epoch {}: lr={:.5g}'.format(
epoch, K.get_value(model.optimizer.lr)))
return K.get_value(model.optimizer.lr)
df_pred_list = []
cv_ext = ''
cv = args.cv if args.cv > 1 else 1
for fold in range(cv):
if args.cv > 1:
logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv))
cv_ext = '.cv{}'.format(fold + 1)
template_model = build_model(loader, args, silent=True)
if args.initial_weights:
logger.info("Loading weights from {}".format(args.initial_weights))
template_model.load_weights(args.initial_weights)
if len(args.gpus) > 1:
from keras.utils import multi_gpu_model
gpu_count = len(args.gpus)
logger.info("Multi GPU with {} gpus".format(gpu_count))
model = multi_gpu_model(template_model,
cpu_merge=False,
gpus=gpu_count)
else:
model = template_model
optimizer = optimizers.deserialize({
'class_name': args.optimizer,
'config': {}
})
base_lr = args.base_lr or K.get_value(optimizer.lr)
if args.learning_rate:
K.set_value(optimizer.lr, args.learning_rate)
model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2])
# calculate trainable and non-trainable params
params.update(candle.compute_trainable_params(model))
candle_monitor = candle.CandleRemoteMonitor(params=params)
timeout_monitor = candle.TerminateOnTimeOut(params['timeout'])
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001)
warmup_lr = LearningRateScheduler(warmup_scheduler)
checkpointer = MultiGPUCheckpoint(prefix + cv_ext + '.model.h5',
save_best_only=True)
tensorboard = TensorBoard(
log_dir="tb/{}{}{}".format(args.tb_prefix, ext, cv_ext))
history_logger = LoggingCallback(logger.debug)
callbacks = [candle_monitor, timeout_monitor, history_logger]
if args.reduce_lr:
callbacks.append(reduce_lr)
if args.warmup_lr:
callbacks.append(warmup_lr)
if args.cp:
callbacks.append(checkpointer)
if args.tb:
callbacks.append(tensorboard)
if args.save_weights:
callbacks.append(
SimpleWeightSaver(args.save_path + '/' + args.save_weights))
if args.use_exported_data is not None:
train_gen = DataFeeder(filename=args.use_exported_data,
batch_size=args.batch_size,
shuffle=args.shuffle,
single=args.single,
agg_dose=args.agg_dose)
val_gen = DataFeeder(partition='val',
filename=args.use_exported_data,
batch_size=args.batch_size,
shuffle=args.shuffle,
single=args.single,
agg_dose=args.agg_dose)
else:
train_gen = CombinedDataGenerator(loader,
fold=fold,
batch_size=args.batch_size,
shuffle=args.shuffle,
single=args.single)
val_gen = CombinedDataGenerator(loader,
partition='val',
fold=fold,
batch_size=args.batch_size,
shuffle=args.shuffle,
single=args.single)
df_val = val_gen.get_response(copy=True)
y_val = df_val[target].values
y_shuf = np.random.permutation(y_val)
log_evaluation(evaluate_prediction(y_val, y_shuf),
description='Between random pairs in y_val:')
if args.no_gen:
x_train_list, y_train = train_gen.get_slice(size=train_gen.size,
single=args.single)
x_val_list, y_val = val_gen.get_slice(size=val_gen.size,
single=args.single)
history = model.fit(x_train_list,
y_train,
batch_size=args.batch_size,
epochs=args.epochs,
callbacks=callbacks,
validation_data=(x_val_list, y_val))
else:
logger.info('Data points per epoch: train = %d, val = %d',
train_gen.size, val_gen.size)
logger.info('Steps per epoch: train = %d, val = %d',
train_gen.steps, val_gen.steps)
history = model.fit_generator(train_gen,
train_gen.steps,
epochs=args.epochs,
callbacks=callbacks,
validation_data=val_gen,
validation_steps=val_gen.steps)
if args.no_gen:
y_val_pred = model.predict(x_val_list, batch_size=args.batch_size)
else:
val_gen.reset()
y_val_pred = model.predict_generator(val_gen, val_gen.steps + 1)
y_val_pred = y_val_pred[:val_gen.size]
y_val_pred = y_val_pred.flatten()
scores = evaluate_prediction(y_val, y_val_pred)
log_evaluation(scores)
# df_val = df_val.assign(PredictedGrowth=y_val_pred, GrowthError=y_val_pred - y_val)
df_val['Predicted' + target] = y_val_pred
df_val[target + 'Error'] = y_val_pred - y_val
df_pred_list.append(df_val)
if hasattr(history, 'loss'):
plot_history(prefix, history, 'loss')
if hasattr(history, 'r2'):
plot_history(prefix, history, 'r2')
pred_fname = prefix + '.predicted.tsv'
df_pred = pd.concat(df_pred_list)
if args.agg_dose:
if args.single:
df_pred.sort_values(['Sample', 'Drug1', target], inplace=True)
else:
df_pred.sort_values(['Source', 'Sample', 'Drug1', 'Drug2', target],
inplace=True)
else:
if args.single:
df_pred.sort_values(['Sample', 'Drug1', 'Dose1', 'Growth'],
inplace=True)
else:
df_pred.sort_values(
['Sample', 'Drug1', 'Drug2', 'Dose1', 'Dose2', 'Growth'],
inplace=True)
df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g')
if args.cv > 1:
scores = evaluate_prediction(df_pred[target],
df_pred['Predicted' + target])
log_evaluation(scores, description='Combining cross validation folds:')
for test_source in loader.test_sep_sources:
test_gen = CombinedDataGenerator(loader,
partition='test',
batch_size=args.batch_size,
source=test_source)
df_test = test_gen.get_response(copy=True)
y_test = df_test[target].values
n_test = len(y_test)
if n_test == 0:
continue
if args.no_gen:
x_test_list, y_test = test_gen.get_slice(size=test_gen.size,
single=args.single)
y_test_pred = model.predict(x_test_list,
batch_size=args.batch_size)
else:
y_test_pred = model.predict_generator(
test_gen.flow(single=args.single), test_gen.steps)
y_test_pred = y_test_pred[:test_gen.size]
y_test_pred = y_test_pred.flatten()
scores = evaluate_prediction(y_test, y_test_pred)
log_evaluation(scores,
description='Testing on data from {} ({})'.format(
test_source, n_test))
if K.backend() == 'tensorflow':
K.clear_session()
logger.handlers = []
return history
def run(params):
args = Struct(**params)
set_seed(args.rng_seed)
ext = extension_from_parameters(args)
verify_path(args.save)
prefix = args.save + ext
logfile = args.logfile if args.logfile else prefix + '.log'
set_up_logger(logfile, args.verbose)
logger.info('Params: {}'.format(params))
loader = CombinedDataLoader(seed=args.rng_seed)
loader.load(
cache=args.cache,
ncols=args.feature_subsample,
cell_features=args.cell_features,
drug_features=args.drug_features,
drug_median_response_min=args.drug_median_response_min,
drug_median_response_max=args.drug_median_response_max,
use_landmark_genes=args.use_landmark_genes,
use_filtered_genes=args.use_filtered_genes,
preprocess_rnaseq=args.preprocess_rnaseq,
single=args.single,
train_sources=args.train_sources,
test_sources=args.test_sources,
embed_feature_source=not args.no_feature_source,
encode_response_source=not args.no_response_source,
)
val_split = args.validation_split
train_split = 1 - val_split
if args.export_data:
fname = args.export_data
loader.partition_data(cv_folds=args.cv,
train_split=train_split,
val_split=val_split,
cell_types=args.cell_types,
by_cell=args.by_cell,
by_drug=args.by_drug)
train_gen = CombinedDataGenerator(loader,
batch_size=args.batch_size,
shuffle=args.shuffle)
val_gen = CombinedDataGenerator(loader,
partition='val',
batch_size=args.batch_size,
shuffle=args.shuffle)
x_train_list, y_train = train_gen.get_slice(size=train_gen.size,
dataframe=True,
single=args.single)
x_val_list, y_val = val_gen.get_slice(size=val_gen.size,
dataframe=True,
single=args.single)
df_train = pd.concat([y_train] + x_train_list, axis=1)
df_val = pd.concat([y_val] + x_val_list, axis=1)
df = pd.concat([df_train, df_val]).reset_index(drop=True)
if args.growth_bins > 1:
df = uno_data.discretize(df, 'Growth', bins=args.growth_bins)
df.to_csv(fname, sep='\t', index=False, float_format="%.3g")
return
loader.partition_data(cv_folds=args.cv,
train_split=train_split,
val_split=val_split,
cell_types=args.cell_types,
by_cell=args.by_cell,
by_drug=args.by_drug)
model = build_model(loader, args)
logger.info('Combined model:')
# model.summary(print_fn=logger.info)
# plot_model(model, to_file=prefix+'.model.png', show_shapes=True)
if args.cp:
model_json = model.to_json()
with open(prefix + '.model.json', 'w') as f:
print(model_json, file=f)
def warmup_scheduler(epoch):
lr = args.learning_rate or base_lr * args.batch_size / 100
if epoch <= 5:
K.set_value(model.optimizer.lr,
(base_lr * (5 - epoch) + lr * epoch) / 5)
logger.debug('Epoch {}: lr={:.5g}'.format(
epoch, K.get_value(model.optimizer.lr)))
return K.get_value(model.optimizer.lr)
df_pred_list = []
cv_ext = ''
cv = args.cv if args.cv > 1 else 1
for fold in range(cv):
if args.cv > 1:
logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv))
cv_ext = '.cv{}'.format(fold + 1)
model = build_model(loader, args, silent=True)
optimizer = optimizers.deserialize({
'class_name': args.optimizer,
'config': {}
})
base_lr = args.base_lr or K.get_value(optimizer.lr)
if args.learning_rate:
K.set_value(optimizer.lr, args.learning_rate)
model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2])
# calculate trainable and non-trainable params
params.update(candle.compute_trainable_params(model))
candle_monitor = candle.CandleRemoteMonitor(params=params)
timeout_monitor = candle.TerminateOnTimeOut(params['timeout'])
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001)
warmup_lr = LearningRateScheduler(warmup_scheduler)
checkpointer = ModelCheckpoint(prefix + cv_ext + '.weights.h5',
save_best_only=True,
save_weights_only=True)
tensorboard = TensorBoard(log_dir="tb/tb{}{}".format(ext, cv_ext))
history_logger = LoggingCallback(logger.debug)
model_recorder = ModelRecorder()
# callbacks = [history_logger, model_recorder]
callbacks = [
candle_monitor, timeout_monitor, history_logger, model_recorder
]
if args.reduce_lr:
callbacks.append(reduce_lr)
if args.warmup_lr:
callbacks.append(warmup_lr)
if args.cp:
callbacks.append(checkpointer)
if args.tb:
callbacks.append(tensorboard)
train_gen = CombinedDataGenerator(loader,
fold=fold,
batch_size=args.batch_size,
shuffle=args.shuffle)
val_gen = CombinedDataGenerator(loader,
partition='val',
fold=fold,
batch_size=args.batch_size,
shuffle=args.shuffle)
df_val = val_gen.get_response(copy=True)
y_val = df_val['Growth'].values
y_shuf = np.random.permutation(y_val)
log_evaluation(evaluate_prediction(y_val, y_shuf),
description='Between random pairs in y_val:')
candleRemoteMonitor = CandleRemoteMonitor(params=params)
callbacks.append(candleRemoteMonitor)
if args.no_gen:
x_train_list, y_train = train_gen.get_slice(size=train_gen.size,
single=args.single)
x_val_list, y_val = val_gen.get_slice(size=val_gen.size,
single=args.single)
history = model.fit(x_train_list,
y_train,
batch_size=args.batch_size,
epochs=args.epochs,
callbacks=callbacks,
validation_data=(x_val_list, y_val))
else:
logger.info('Data points per epoch: train = %d, val = %d',
train_gen.size, val_gen.size)
logger.info('Steps per epoch: train = %d, val = %d',
train_gen.steps, val_gen.steps)
history = model.fit_generator(
train_gen.flow(single=args.single),
train_gen.steps,
epochs=args.epochs,
callbacks=callbacks,
validation_data=val_gen.flow(single=args.single),
validation_steps=val_gen.steps)
if args.cp:
model.load_weights(prefix + cv_ext + '.weights.h5')
# model = model_recorder.best_model
if args.no_gen:
y_val_pred = model.predict(x_val_list, batch_size=args.batch_size)
else:
val_gen.reset()
y_val_pred = model.predict_generator(
val_gen.flow(single=args.single), val_gen.steps)
y_val_pred = y_val_pred[:val_gen.size]
y_val_pred = y_val_pred.flatten()
scores = evaluate_prediction(y_val, y_val_pred)
log_evaluation(scores)
df_val = df_val.assign(PredictedGrowth=y_val_pred,
GrowthError=y_val_pred - y_val)
df_pred_list.append(df_val)
plot_history(prefix, history, 'loss')
plot_history(prefix, history, 'r2')
pred_fname = prefix + '.predicted.tsv'
df_pred = pd.concat(df_pred_list)
df_pred.sort_values(
['Source', 'Sample', 'Drug1', 'Drug2', 'Dose1', 'Dose2', 'Growth'],
inplace=True)
df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g')
if args.cv > 1:
scores = evaluate_prediction(df_pred['Growth'],
df_pred['PredictedGrowth'])
log_evaluation(scores, description='Combining cross validation folds:')
for test_source in loader.test_sep_sources:
test_gen = CombinedDataGenerator(loader,
partition='test',
batch_size=args.batch_size,
source=test_source)
df_test = test_gen.get_response(copy=True)
y_test = df_test['Growth'].values
n_test = len(y_test)
if n_test == 0:
continue
if args.no_gen:
x_test_list, y_test = test_gen.get_slice(size=test_gen.size,
single=args.single)
y_test_pred = model.predict(x_test_list,
batch_size=args.batch_size)
else:
y_test_pred = model.predict_generator(
test_gen.flow(single=args.single), test_gen.steps)
y_test_pred = y_test_pred[:test_gen.size]
y_test_pred = y_test_pred.flatten()
scores = evaluate_prediction(y_test, y_test_pred)
log_evaluation(scores,
description='Testing on data from {} ({})'.format(
test_source, n_test))
if K.backend() == 'tensorflow':
K.clear_session()
logger.handlers = []
return history
def run(params):
args = candle.ArgumentStruct(**params)
seed = args.rng_seed
candle.set_seed(seed)
# Construct extension to save model
ext = p1b1.extension_from_parameters(params, '.keras')
candle.verify_path(params['save'])
prefix = '{}{}'.format(params['save'], ext)
logfile = params['logfile'] if params['logfile'] else prefix + '.log'
candle.set_up_logger(logfile, p1b1.logger, params['verbose'])
p1b1.logger.info('Params: {}'.format(params))
# Get default parameters for initialization and optimizer functions
keras_defaults = candle.keras_default_config()
# Load dataset
x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels = p1b1.load_data(
params, seed)
# cache_file = 'data_l1000_cache.h5'
# save_cache(cache_file, x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels)
# x_train, y_train, x_val, y_val, x_test, y_test, x_labels, y_labels = load_cache(cache_file)
p1b1.logger.info("Shape x_train: {}".format(x_train.shape))
p1b1.logger.info("Shape x_val: {}".format(x_val.shape))
p1b1.logger.info("Shape x_test: {}".format(x_test.shape))
p1b1.logger.info("Range x_train: [{:.3g}, {:.3g}]".format(
np.min(x_train), np.max(x_train)))
p1b1.logger.info("Range x_val: [{:.3g}, {:.3g}]".format(
np.min(x_val), np.max(x_val)))
p1b1.logger.info("Range x_test: [{:.3g}, {:.3g}]".format(
np.min(x_test), np.max(x_test)))
p1b1.logger.debug('Class labels')
for i, label in enumerate(y_labels):
p1b1.logger.debug(' {}: {}'.format(i, label))
# clf = build_type_classifier(x_train, y_train, x_val, y_val)
n_classes = len(y_labels)
cond_train = y_train
cond_val = y_val
cond_test = y_test
input_dim = x_train.shape[1]
cond_dim = cond_train.shape[1]
latent_dim = params['latent_dim']
activation = params['activation']
dropout = params['drop']
dense_layers = params['dense']
dropout_layer = keras.layers.noise.AlphaDropout if params[
'alpha_dropout'] else Dropout
# Initialize weights and learning rule
initializer_weights = candle.build_initializer(params['initialization'],
keras_defaults, seed)
initializer_bias = candle.build_initializer('constant', keras_defaults, 0.)
if dense_layers is not None:
if type(dense_layers) != list:
dense_layers = list(dense_layers)
else:
dense_layers = []
# Encoder Part
x_input = Input(shape=(input_dim, ))
cond_input = Input(shape=(cond_dim, ))
h = x_input
if params['model'] == 'cvae':
h = keras.layers.concatenate([x_input, cond_input])
for i, layer in enumerate(dense_layers):
if layer > 0:
x = h
h = Dense(layer,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias)(h)
if params['residual']:
try:
h = keras.layers.add([h, x])
except ValueError:
pass
if params['batch_normalization']:
h = BatchNormalization()(h)
if dropout > 0:
h = dropout_layer(dropout)(h)
if params['model'] == 'ae':
encoded = Dense(latent_dim,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias)(h)
else:
epsilon_std = params['epsilon_std']
z_mean = Dense(latent_dim, name='z_mean')(h)
z_log_var = Dense(latent_dim, name='z_log_var')(h)
encoded = z_mean
def vae_loss(x, x_decoded_mean):
xent_loss = binary_crossentropy(x, x_decoded_mean)
kl_loss = -0.5 * K.sum(
1 + z_log_var - K.square(z_mean) - K.exp(z_log_var), axis=-1)
return K.mean(xent_loss + kl_loss / input_dim)
def sampling(params):
z_mean_, z_log_var_ = params
batch_size = K.shape(z_mean_)[0]
epsilon = K.random_normal(shape=(batch_size, latent_dim),
mean=0.,
stddev=epsilon_std)
return z_mean_ + K.exp(z_log_var_ / 2) * epsilon
z = Lambda(sampling, output_shape=(latent_dim, ))([z_mean, z_log_var])
if params['model'] == 'cvae':
z_cond = keras.layers.concatenate([z, cond_input])
# Decoder Part
decoder_input = Input(shape=(latent_dim, ))
h = decoder_input
if params['model'] == 'cvae':
h = keras.layers.concatenate([decoder_input, cond_input])
for i, layer in reversed(list(enumerate(dense_layers))):
if layer > 0:
x = h
h = Dense(layer,
activation=activation,
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias)(h)
if params['residual']:
try:
h = keras.layers.add([h, x])
except ValueError:
pass
if params['batch_normalization']:
h = BatchNormalization()(h)
if dropout > 0:
h = dropout_layer(dropout)(h)
decoded = Dense(input_dim,
activation='sigmoid',
kernel_initializer=initializer_weights,
bias_initializer=initializer_bias)(h)
# Build autoencoder model
if params['model'] == 'cvae':
encoder = Model([x_input, cond_input], encoded)
decoder = Model([decoder_input, cond_input], decoded)
model = Model([x_input, cond_input], decoder([z, cond_input]))
loss = vae_loss
metrics = [xent, corr, mse]
elif params['model'] == 'vae':
encoder = Model(x_input, encoded)
decoder = Model(decoder_input, decoded)
model = Model(x_input, decoder(z))
loss = vae_loss
metrics = [xent, corr, mse]
else:
encoder = Model(x_input, encoded)
decoder = Model(decoder_input, decoded)
model = Model(x_input, decoder(encoded))
loss = params['loss']
metrics = [xent, corr]
model.summary()
decoder.summary()
if params['cp']:
model_json = model.to_json()
with open(prefix + '.model.json', 'w') as f:
print(model_json, file=f)
# Define optimizer
# optimizer = candle.build_optimizer(params['optimizer'],
# params['learning_rate'],
# keras_defaults)
optimizer = optimizers.deserialize({
'class_name': params['optimizer'],
'config': {}
})
base_lr = params['base_lr'] or K.get_value(optimizer.lr)
if params['learning_rate']:
K.set_value(optimizer.lr, params['learning_rate'])
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
# calculate trainable and non-trainable params
params.update(candle.compute_trainable_params(model))
def warmup_scheduler(epoch):
lr = params['learning_rate'] or base_lr * params['batch_size'] / 100
if epoch <= 5:
K.set_value(model.optimizer.lr,
(base_lr * (5 - epoch) + lr * epoch) / 5)
p1b1.logger.debug('Epoch {}: lr={}'.format(
epoch, K.get_value(model.optimizer.lr)))
return K.get_value(model.optimizer.lr)
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001)
warmup_lr = LearningRateScheduler(warmup_scheduler)
checkpointer = ModelCheckpoint(params['save'] + ext + '.weights.h5',
save_best_only=True,
save_weights_only=True)
tensorboard = TensorBoard(log_dir="tb/tb{}".format(ext))
candle_monitor = candle.CandleRemoteMonitor(params=params)
timeout_monitor = candle.TerminateOnTimeOut(params['timeout'])
history_logger = LoggingCallback(p1b1.logger.debug)
callbacks = [candle_monitor, timeout_monitor, history_logger]
if params['reduce_lr']:
callbacks.append(reduce_lr)
if params['warmup_lr']:
callbacks.append(warmup_lr)
if params['cp']:
callbacks.append(checkpointer)
if params['tb']:
callbacks.append(tensorboard)
x_val2 = np.copy(x_val)
np.random.shuffle(x_val2)
start_scores = p1b1.evaluate_autoencoder(x_val, x_val2)
p1b1.logger.info('\nBetween random pairs of validation samples: {}'.format(
start_scores))
if params['model'] == 'cvae':
inputs = [x_train, cond_train]
val_inputs = [x_val, cond_val]
test_inputs = [x_test, cond_test]
else:
inputs = x_train
val_inputs = x_val
test_inputs = x_test
outputs = x_train
val_outputs = x_val
test_outputs = x_test
history = model.fit(inputs,
outputs,
verbose=2,
batch_size=params['batch_size'],
epochs=params['epochs'],
callbacks=callbacks,
validation_data=(val_inputs, val_outputs))
if params['cp']:
encoder.save(prefix + '.encoder.h5')
decoder.save(prefix + '.decoder.h5')
plot_history(prefix, history, 'loss')
plot_history(prefix, history, 'corr', 'streaming pearson correlation')
# Evalute model on test set
x_pred = model.predict(test_inputs)
scores = p1b1.evaluate_autoencoder(x_pred, x_test)
p1b1.logger.info('\nEvaluation on test data: {}'.format(scores))
x_test_encoded = encoder.predict(test_inputs,
batch_size=params['batch_size'])
y_test_classes = np.argmax(y_test, axis=1)
plot_scatter(x_test_encoded, y_test_classes, prefix + '.latent')
if params['tsne']:
tsne = TSNE(n_components=2, random_state=seed)
x_test_encoded_tsne = tsne.fit_transform(x_test_encoded)
plot_scatter(x_test_encoded_tsne, y_test_classes,
prefix + '.latent.tsne')
# diff = x_pred - x_test
# plt.hist(diff.ravel(), bins='auto')
# plt.title("Histogram of Errors with 'auto' bins")
# plt.savefig('histogram_keras.png')
# generate synthetic data
# epsilon_std = 1.0
# for i in range(1000):
# z_sample = np.random.normal(size=(1, 2)) * epsilon_std
# x_decoded = decoder.predict(z_sample)
p1b1.logger.handlers = []
return history
def load_model(filepath, custom_objects=None, compile=True):
"""Loads an Evolutron model saved via Model.save().
# Arguments
filepath: String, path to the saved model.
custom_objects: Optional dictionary mapping names
(strings) to custom classes or functions to be
considered during deserialization.
compile: Boolean, whether to compile the model
after loading.
# Returns
An Evolutron model instance. If an optimizer was found
as part of the saved model, the model is already
compiled. Otherwise, the model is uncompiled and
a warning will be displayed. When `compile` is set
to False, the compilation is omitted without any
warning.
# Raises
ImportError: if h5py is not available.
ValueError: In case of an invalid savefile.
"""
if h5py is None:
raise ImportError('`load_model` requires h5py.')
if not custom_objects:
custom_objects = {}
def convert_custom_objects(obj):
"""Handles custom object lookup.
# Arguments
obj: object, dict, or list.
# Returns
The same structure, where occurences
of a custom object name have been replaced
with the custom object.
"""
if isinstance(obj, list):
deserialized = []
for value in obj:
if value in custom_objects:
deserialized.append(custom_objects[value])
else:
deserialized.append(value)
return deserialized
if isinstance(obj, dict):
deserialized = {}
for key, value in obj.items():
deserialized[key] = []
if isinstance(value, list):
for element in value:
if element in custom_objects:
deserialized[key].append(custom_objects[element])
else:
deserialized[key].append(element)
elif value in custom_objects:
deserialized[key] = custom_objects[value]
else:
deserialized[key] = value
return deserialized
if obj in custom_objects:
return custom_objects[obj]
return obj
f = h5py.File(filepath, mode='r')
# instantiate model
model_config = f.attrs.get('model_config')
if model_config is None:
raise ValueError('No model found in config file.')
model_config = json.loads(model_config.decode('utf-8'))
globs = globals() # All layers.
globs['Model'] = Model
model = deserialize_keras_object(model_config,
module_objects=globs,
custom_objects=custom_objects,
printable_module_name='layer')
# set weights
topology.load_weights_from_hdf5_group(f['model_weights'], model.layers)
# Early return if compilation is not required.
if not compile:
f.close()
return model
# instantiate optimizer
training_config = f.attrs.get('training_config')
if training_config is None:
warnings.warn('No training configuration found in save file: '
'the model was *not* compiled. Compile it manually.')
f.close()
return model
training_config = json.loads(training_config.decode('utf-8'))
optimizer_config = training_config['optimizer_config']
optimizer = opt.deserialize(optimizer_config,
custom_objects=custom_objects)
# Recover loss functions and metrics.
loss = convert_custom_objects(training_config['loss'])
metrics = convert_custom_objects(training_config['metrics'])
sample_weight_mode = training_config['sample_weight_mode']
loss_weights = training_config['loss_weights']
# Compile model.
model.compile(optimizer=optimizer,
loss=loss,
metrics=metrics,
loss_weights=loss_weights,
sample_weight_mode=sample_weight_mode)
# Set optimizer weights.
if 'optimizer_weights' in f:
# Build train function (to get weight updates).
model._make_train_function()
optimizer_weights_group = f['optimizer_weights']
optimizer_weight_names = [
n.decode('utf8')
for n in optimizer_weights_group.attrs['weight_names']
]
optimizer_weight_values = [
optimizer_weights_group[n] for n in optimizer_weight_names
]
model.optimizer.set_weights(optimizer_weight_values)
f.close()
return model
def from_config(cls, config, custom_objects=None):
optimizer_config = config.pop('optimizer')
optimizer = deserialize(optimizer_config)
return cls(optimizer=optimizer, **config)
def run(params):
args = Struct(**params)
set_seed(args.rng_seed)
ext = extension_from_parameters(args)
prefix = args.save + ext
logfile = args.logfile if args.logfile else prefix + '.log'
set_up_logger(logfile, args.verbose)
logger.info('Params: {}'.format(params))
loader = ComboDataLoader(seed=args.rng_seed,
val_split=args.validation_split,
cell_features=args.cell_features,
drug_features=args.drug_features,
use_landmark_genes=args.use_landmark_genes,
use_combo_score=args.use_combo_score,
cv_partition=args.cv_partition,
cv=args.cv)
# test_loader(loader)
# test_generator(loader)
train_gen = ComboDataGenerator(loader, batch_size=args.batch_size).flow()
val_gen = ComboDataGenerator(loader,
partition='val',
batch_size=args.batch_size).flow()
train_steps = int(loader.n_train / args.batch_size)
val_steps = int(loader.n_val / args.batch_size)
model = build_model(loader, args, verbose=True)
model.summary()
# plot_model(model, to_file=prefix+'.model.png', show_shapes=True)
if args.cp:
model_json = model.to_json()
with open(prefix + '.model.json', 'w') as f:
print(model_json, file=f)
def warmup_scheduler(epoch):
lr = args.learning_rate or base_lr * args.batch_size / 100
if epoch <= 5:
K.set_value(model.optimizer.lr,
(base_lr * (5 - epoch) + lr * epoch) / 5)
logger.debug('Epoch {}: lr={}'.format(epoch,
K.get_value(model.optimizer.lr)))
return K.get_value(model.optimizer.lr)
df_pred_list = []
cv_ext = ''
cv = args.cv if args.cv > 1 else 1
fold = 0
while fold < cv:
if args.cv > 1:
logger.info('Cross validation fold {}/{}:'.format(fold + 1, cv))
cv_ext = '.cv{}'.format(fold + 1)
model = build_model(loader, args)
optimizer = optimizers.deserialize({
'class_name': args.optimizer,
'config': {}
})
base_lr = args.base_lr or K.get_value(optimizer.lr)
if args.learning_rate:
K.set_value(optimizer.lr, args.learning_rate)
model.compile(loss=args.loss, optimizer=optimizer, metrics=[mae, r2])
# calculate trainable and non-trainable params
params.update(compute_trainable_params(model))
candle_monitor = CandleRemoteMonitor(params=params)
timeout_monitor = TerminateOnTimeOut(params['timeout'])
reduce_lr = ReduceLROnPlateau(monitor='val_loss',
factor=0.5,
patience=5,
min_lr=0.00001)
warmup_lr = LearningRateScheduler(warmup_scheduler)
checkpointer = ModelCheckpoint(prefix + cv_ext + '.weights.h5',
save_best_only=True,
save_weights_only=True)
tensorboard = TensorBoard(log_dir="tb/tb{}{}".format(ext, cv_ext))
history_logger = LoggingCallback(logger.debug)
model_recorder = ModelRecorder()
# callbacks = [history_logger, model_recorder]
callbacks = [
candle_monitor, timeout_monitor, history_logger, model_recorder
]
if args.reduce_lr:
callbacks.append(reduce_lr)
if args.warmup_lr:
callbacks.append(warmup_lr)
if args.cp:
callbacks.append(checkpointer)
if args.tb:
callbacks.append(tensorboard)
if args.gen:
history = model.fit_generator(train_gen,
train_steps,
epochs=args.epochs,
callbacks=callbacks,
validation_data=val_gen,
validation_steps=val_steps)
else:
if args.cv > 1:
x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data_cv(
fold)
else:
x_train_list, y_train, x_val_list, y_val, df_train, df_val = loader.load_data(
)
y_shuf = np.random.permutation(y_val)
log_evaluation(evaluate_prediction(y_val, y_shuf),
description='Between random pairs in y_val:')
history = model.fit(x_train_list,
y_train,
batch_size=args.batch_size,
shuffle=args.shuffle,
epochs=args.epochs,
callbacks=callbacks,
validation_data=(x_val_list, y_val))
if args.cp:
model.load_weights(prefix + cv_ext + '.weights.h5')
if not args.gen:
y_val_pred = model.predict(x_val_list,
batch_size=args.batch_size).flatten()
scores = evaluate_prediction(y_val, y_val_pred)
if args.cv > 1 and scores[args.loss] > args.max_val_loss:
logger.warn(
'Best val_loss {} is greater than {}; retrain the model...'
.format(scores[args.loss], args.max_val_loss))
continue
else:
fold += 1
log_evaluation(scores)
df_val.is_copy = False
df_val['GROWTH_PRED'] = y_val_pred
df_val['GROWTH_ERROR'] = y_val_pred - y_val
df_pred_list.append(df_val)
if args.cp:
# model.save(prefix+'.model.h5')
model_recorder.best_model.save(prefix + '.model.h5')
# test reloadded model prediction
new_model = keras.models.load_model(prefix + '.model.h5')
new_model.load_weights(prefix + cv_ext + '.weights.h5')
new_pred = new_model.predict(x_val_list,
batch_size=args.batch_size).flatten()
# print('y_val:', y_val[:10])
# print('old_pred:', y_val_pred[:10])
# print('new_pred:', new_pred[:10])
plot_history(prefix, history, 'loss')
plot_history(prefix, history, 'r2')
if K.backend() == 'tensorflow':
K.clear_session()
pred_fname = prefix + '.predicted.growth.tsv'
if args.use_combo_score:
pred_fname = prefix + '.predicted.score.tsv'
df_pred = pd.concat(df_pred_list)
df_pred.to_csv(pred_fname, sep='\t', index=False, float_format='%.4g')
logger.handlers = []
return history