def from_keras(cls, keras_model, dataset):
import tensorflow.keras.backend as keras_backend
loss = keras_model.total_loss
inputs = keras_model.inputs + keras_model.targets + keras_model.sample_weights
variables = keras_model._collected_trainable_weights
keras_optimizer = keras_model.optimizer
grads = keras_optimizer.get_gradients(loss, variables)
sess = keras_backend.get_session()
with sess.as_default():
optim_method = OptimConverter.to_bigdl_optim_method(
keras_optimizer)
if keras_model.metrics:
if isinstance(keras_model.metrics, dict):
raise ValueError(
"different metrics for different outputs are not supported right now"
)
bigdl_val_methods = [
to_bigdl_metric(m) for m in keras_model.metrics_names
]
val_outputs = keras_model.outputs
val_labels = keras_model.targets
else:
val_outputs = None
val_labels = None
bigdl_val_methods = None
return cls(loss, optim_method, sess, dataset, inputs, grads, variables,
loss.graph, val_outputs, val_labels, bigdl_val_methods,
len(keras_model.sample_weights))
def from_keras(cls, keras_model, dataset, optim_method=None, val_spilt=0.0, **kwargs):
import tensorflow.keras.backend as K
loss = keras_model.total_loss
inputs = keras_model.inputs + keras_model.targets
variables = keras_model._collected_trainable_weights
keras_optimizer = keras_model.optimizer
grads = K.gradients(loss, variables)
if None in grads:
raise ValueError('An operation has `None` for gradient. '
'Please make sure that all of your ops have a '
'gradient defined (i.e. are differentiable). '
'Common ops without gradient: '
'K.argmax, K.round, K.eval.')
clip_norm = None
clip_value = None
if hasattr(keras_optimizer, 'clipnorm'):
clip_norm = keras_optimizer.clipnorm
if hasattr(keras_optimizer, 'clipvalue'):
clip_value = (-keras_optimizer.clipvalue, keras_optimizer.clipvalue)
sess = K.get_session()
if optim_method is None:
optim_method = keras_optimizer
optim_method = TFOptimizer.to_bigdl_optim_method(optim_method)
if keras_model.metrics and (dataset.get_validation_data() is not None or val_spilt != 0.0):
if isinstance(keras_model.metrics, dict):
raise ValueError(
"different metrics for different outputs are not supported right now")
if dataset.get_validation_data() is None and val_spilt == 0.0:
raise ValueError("Validation data is not specified. Please set " +
"val_rdd in TFDataset, or set val_split larger than zero")
bigdl_val_methods = \
[to_bigdl_metric(m, keras_model.loss) for m in keras_model.metrics_names]
val_outputs = keras_model.outputs
val_labels = keras_model.targets
else:
val_outputs = None
val_labels = None
bigdl_val_methods = None
tensor_with_value = {
K.learning_phase(): [True, False]
}
return cls(loss, optim_method, sess, dataset, inputs,
grads, variables, loss.graph, val_outputs, val_labels,
bigdl_val_methods, val_spilt,
tensors_with_value=tensor_with_value,
clip_norm=clip_norm,
clip_value=clip_value, **kwargs)
def from_keras(cls, keras_model, dataset, val_spilt=0.0, **kwargs):
import tensorflow.keras.backend as K
loss = keras_model.total_loss
inputs = keras_model.inputs + keras_model.targets
variables = keras_model._collected_trainable_weights
keras_optimizer = keras_model.optimizer
grads = keras_optimizer.get_gradients(loss, variables)
sess = K.get_session()
with sess.as_default():
optim_method = TFOptimizer.to_bigdl_optim_method(keras_optimizer)
if keras_model.metrics:
if isinstance(keras_model.metrics, dict):
raise ValueError(
"different metrics for different outputs are not supported right now"
)
if dataset.val_rdd is None and val_spilt == 0.0:
raise ValueError(
"Validation data is not specified. Please set " +
"val rdd in TFDataset, or set val_split larger than zero")
bigdl_val_methods = [
to_bigdl_metric(m) for m in keras_model.metrics_names
]
val_outputs = keras_model.outputs
val_labels = keras_model.targets
else:
val_outputs = None
val_labels = None
bigdl_val_methods = None
tensor_with_value = {K.learning_phase(): [True, False]}
return cls(loss,
optim_method,
sess,
dataset,
inputs,
grads,
variables,
loss.graph,
val_outputs,
val_labels,
bigdl_val_methods,
val_spilt,
tensors_with_value=tensor_with_value,
**kwargs)
def from_keras(cls,
keras_model,
dataset,
optim_method=None,
val_spilt=0.0,
**kwargs):
"""
Create a TFOptimizer from a tensorflow.keras model. The model must be compiled.
:param keras_model: the tensorflow.keras model, which must be compiled.
:param dataset: a TFDataset
:param optim_method: the optimization method to be used, such as bigdl.optim.optimizer.Adam
:param val_spilt: Float between 0 and 1. Fraction of the training data to be used as
validation data.
:return:
"""
import tensorflow.keras.backend as K
loss = keras_model.total_loss
model_inputs = keras_model.inputs
if hasattr(keras_model, "targets"):
model_targets = keras_model.targets
else:
model_targets = keras_model._targets
inputs = model_inputs + model_targets
variables = keras_model._collected_trainable_weights
variables.sort(key=lambda variable: variable.name)
keras_optimizer = keras_model.optimizer
grads = K.gradients(loss, variables)
if None in grads:
raise ValueError('An operation has `None` for gradient. '
'Please make sure that all of your ops have a '
'gradient defined (i.e. are differentiable). '
'Common ops without gradient: '
'K.argmax, K.round, K.eval.')
clip_norm = None
clip_value = None
if hasattr(keras_optimizer, 'clipnorm'):
clip_norm = keras_optimizer.clipnorm
if hasattr(keras_optimizer, 'clipvalue'):
clip_value = (-keras_optimizer.clipvalue,
keras_optimizer.clipvalue)
sess = K.get_session()
if optim_method is None:
optim_method = keras_optimizer
optim_method = to_bigdl_optim_method(optim_method)
if keras_model.metrics and (dataset.get_validation_data() is not None
or val_spilt != 0.0):
if isinstance(keras_model.metrics, dict):
raise ValueError(
"different metrics for different outputs are not supported right now"
)
if dataset.get_validation_data() is None and val_spilt == 0.0:
raise ValueError(
"Validation data is not specified. Please set " +
"val_rdd in TFDataset, or set val_split larger than zero")
if len(keras_model.outputs) > 1:
if not all([
name.endswith("loss")
for name in keras_model.metrics_names
]):
raise ValueError(
"metrics (except loss) for multi-head model is not supported"
)
else:
bigdl_val_methods = [Loss()]
val_outputs = keras_model.outputs
val_labels = model_targets
else:
bigdl_val_methods = \
[to_bigdl_metric(m, keras_model.loss) for m in keras_model.metrics_names]
val_outputs = keras_model.outputs
val_labels = model_targets
else:
val_outputs = None
val_labels = None
bigdl_val_methods = None
tensor_with_value = {K.learning_phase(): [True, False]}
updates = keras_model.updates
return cls(loss,
optim_method,
sess,
dataset,
inputs,
grads,
variables,
loss.graph,
val_outputs,
val_labels,
bigdl_val_methods,
val_spilt,
tensors_with_value=tensor_with_value,
clip_norm=clip_norm,
clip_value=clip_value,
updates=updates,
**kwargs)
def from_keras(cls,
keras_model,
dataset,
session_config=None,
model_dir=None,
metrics=None,
optimizer=None):
"""
Create a TFOptimizer from a tensorflow.keras model. The model must be compiled.
:param keras_model: the tensorflow.keras model, which must be compiled.
:param dataset: a TFDataset
:return:
"""
import tensorflow.keras.backend as K
model_inputs = keras_model.inputs
if hasattr(keras_model, "targets"):
model_targets = keras_model.targets
else:
model_targets = keras_model._targets
# target can be None if loss is None
model_targets = list(filter(lambda x: x is not None, model_targets))
# standarize feature, labels to support keras model
if isinstance(dataset, TFNdarrayDataset):
dataset = _standarize_feature_label_dataset(dataset, keras_model)
flatten_inputs = nest.flatten(dataset.feature_tensors)
assert len(model_inputs) == len(flatten_inputs), \
("the keras model and TFDataset should have the same number of tensors" +
" keras model has {} inputs " +
"while TFDataset has {} inputs").format(len(model_inputs),
len(flatten_inputs))
for i in range(len(flatten_inputs)):
if not TFOptimizer._shape_match(model_inputs[i].shape,
flatten_inputs[i].shape):
raise ValueError(("The {}th input in keras model {}"
" does not match the TFDataset"
"input {}").format(i, model_inputs[i],
flatten_inputs[i]))
flatten_targets = nest.flatten(dataset.label_tensors)
assert len(model_targets) == len(flatten_targets), \
("the keras model and TFDataset should have the same number of tensors" +
" keras model has {} targets " +
"while TFDataset has {} labels").format(len(model_targets),
len(flatten_inputs))
# todo check targets shape, currently checking target shape will
# cause too much false alarm.
loss = keras_model.total_loss
variables = keras_model._collected_trainable_weights
variables.sort(key=lambda variable: variable.name)
keras_optimizer = keras_model.optimizer
from zoo.tfpark.zoo_optimizer import get_gradients_for_keras
grads = get_gradients_for_keras(keras_optimizer, loss, variables)
grads_and_vars = list(zip(grads, variables))
import tensorflow.python.keras.optimizers as koptimizers
if isinstance(keras_optimizer, koptimizers.TFOptimizer):
# work around keras TFOptimzier bug
train_op = keras_optimizer.optimizer.apply_gradients(
grads_and_vars)
else:
train_op = keras_optimizer.apply_gradients(grads_and_vars)
sess = K.get_session()
if keras_model.metrics and (dataset.get_validation_data() is not None):
if isinstance(keras_model.metrics, dict):
raise ValueError(
"different metrics for different outputs are not supported right now"
)
if len(keras_model.outputs) > 1:
if not all([
name.endswith("loss")
for name in keras_model.metrics_names
]):
raise ValueError(
"metrics (except loss) for multi-head model is not supported"
)
else:
bigdl_val_methods = [Loss()]
val_outputs = keras_model.outputs
val_labels = model_targets
else:
bigdl_val_methods = \
[to_bigdl_metric(m, keras_model.loss) for m in keras_model.metrics_names]
val_outputs = keras_model.outputs
val_labels = model_targets
else:
val_outputs = None
val_labels = None
bigdl_val_methods = None
tensor_with_value = {K.learning_phase(): [True, False]}
updates = []
updates += keras_model.get_updates_for(None)
# Conditional updates relevant to this model
updates += keras_model.get_updates_for(keras_model.inputs)
if bigdl_val_methods is not None:
val_methods = to_list(bigdl_val_methods)
bigdl_metrics = {}
for i, method in enumerate(val_methods):
bigdl_metrics['bigdl_metric_' + str(i)] = BigDLMetric(
method, val_outputs, val_labels)
if metrics is None:
metrics = bigdl_metrics
else:
metrics.update(bigdl_metrics)
if optimizer is not None:
clip_norm = None
clip_value = None
if hasattr(keras_optimizer, 'clipnorm'):
clip_norm = keras_optimizer.clipnorm
if hasattr(keras_optimizer, 'clipvalue'):
clip_value = (-keras_optimizer.clipvalue,
keras_optimizer.clipvalue)
tf_model = TFModel.create(loss,
sess,
model_inputs,
model_targets,
keras_model.outputs,
grads,
variables,
loss.graph,
tensor_with_value,
session_config,
metrics,
updates,
model_dir=None)
return cls(tf_model,
optimizer,
sess=sess,
dataset=dataset,
clip_norm=clip_norm,
clip_value=clip_value,
model_dir=model_dir)
return cls.from_train_op(train_op,
loss,
inputs=model_inputs,
labels=model_targets,
metrics=metrics,
updates=updates,
sess=sess,
dataset=dataset,
tensor_with_value=tensor_with_value,
session_config=session_config,
model_dir=model_dir)
def from_keras(cls, keras_model, dataset, optim_method=None,
session_config=None, model_dir=None):
"""
Create a TFOptimizer from a tensorflow.keras model. The model must be compiled.
:param keras_model: the tensorflow.keras model, which must be compiled.
:param dataset: a TFDataset
:param optim_method: the optimization method to be used, such as bigdl.optim.optimizer.Adam
validation data.
:return:
"""
import tensorflow.keras.backend as K
model_inputs = keras_model.inputs
if hasattr(keras_model, "targets"):
model_targets = keras_model.targets
else:
model_targets = keras_model._targets
flatten_inputs = nest.flatten(dataset.feature_tensors)
assert len(model_inputs) == len(flatten_inputs), \
("the keras model and TFDataset should have the same number of tensors" +
" keras model has {} inputs " +
"while TFDataset has {} inputs").format(len(model_inputs),
len(flatten_inputs))
for i in range(len(flatten_inputs)):
if not TFOptimizer._shape_match(model_inputs[i].shape, flatten_inputs[i].shape):
raise ValueError(("The {}th input in keras model {}"
" does not match the TFDataset"
"input {}").format(i,
model_inputs[i],
flatten_inputs[i]))
flatten_targets = nest.flatten(dataset.label_tensors)
assert len(model_targets) == len(flatten_targets), \
("the keras model and TFDataset should have the same number of tensors" +
" keras model has {} targets " +
"while TFDataset has {} labels").format(len(model_targets),
len(flatten_inputs))
# todo check targets shape, currently checking target shape will
# cause too much false alarm.
loss = keras_model.total_loss
variables = keras_model._collected_trainable_weights
variables.sort(key=lambda variable: variable.name)
keras_optimizer = keras_model.optimizer
grads = K.gradients(loss, variables)
if None in grads:
raise ValueError('An operation has `None` for gradient. '
'Please make sure that all of your ops have a '
'gradient defined (i.e. are differentiable). '
'Common ops without gradient: '
'K.argmax, K.round, K.eval.')
clip_norm = None
clip_value = None
if hasattr(keras_optimizer, 'clipnorm'):
clip_norm = keras_optimizer.clipnorm
if hasattr(keras_optimizer, 'clipvalue'):
clip_value = (-keras_optimizer.clipvalue, keras_optimizer.clipvalue)
sess = K.get_session()
if optim_method is None:
optim_method = keras_optimizer
optim_method = to_bigdl_optim_method(optim_method)
if keras_model.metrics and (dataset.get_validation_data() is not None):
if isinstance(keras_model.metrics, dict):
raise ValueError(
"different metrics for different outputs are not supported right now")
if dataset.get_validation_data() is None:
raise ValueError("Validation data is not specified. Please set " +
"val_rdd in TFDataset")
if len(keras_model.outputs) > 1:
if not all([name.endswith("loss") for name in keras_model.metrics_names]):
raise ValueError("metrics (except loss) for multi-head model is not supported")
else:
bigdl_val_methods = [Loss()]
val_outputs = keras_model.outputs
val_labels = model_targets
else:
bigdl_val_methods = \
[to_bigdl_metric(m, keras_model.loss) for m in keras_model.metrics_names]
val_outputs = keras_model.outputs
val_labels = model_targets
else:
val_outputs = None
val_labels = None
bigdl_val_methods = None
tensor_with_value = {
K.learning_phase(): [True, False]
}
updates = keras_model.updates
metrics = None
if bigdl_val_methods is not None:
val_methods = to_list(bigdl_val_methods)
metrics = {}
for i, method in enumerate(val_methods):
metrics['bigdl_metirc_' + str(i)] = BigDLMetric(method, val_outputs, val_labels)
tf_model = TFModel.create(loss, sess, model_inputs, model_targets, keras_model.outputs,
grads, variables, loss.graph,
tensor_with_value, session_config, metrics,
updates, model_dir=None)
return cls(tf_model, optim_method, sess=sess, dataset=dataset,
clip_norm=clip_norm, clip_value=clip_value, model_dir=model_dir)
