def keras_evaluate(keras_model, eval_dataset_fn, save, keras_model_pkg,
validation_metrics):
model_metrics = []
if hasattr(keras_model_pkg, "eval_metrics_fn"):
metrics_functions = keras_model_pkg.eval_metrics_fn()
for key, func in metrics_functions.items():
func.__name__ = key
model_metrics.append(func)
# use WITH specified metrics if it's not default.
if validation_metrics != ["Accuracy"]:
keras_metrics = metrics.get_keras_metrics(validation_metrics)
else:
if len(model_metrics) > 0:
keras_metrics = model_metrics
else:
# default
keras_metrics = metrics.get_keras_metrics(["Accuracy"])
has_custom_evaluate_func = hasattr(keras_model, 'sqlflow_evaluate_loop')
if not has_custom_evaluate_func:
# compile the model with default arguments only for evaluation
# (run forward only).
keras_model.compile(loss=keras_model_pkg.loss, metrics=keras_metrics)
eval_dataset = eval_dataset_fn()
def get_features(sample, label):
return sample
eval_dataset_x = eval_dataset.map(get_features)
if has_custom_evaluate_func:
result = keras_model.sqlflow_evaluate_loop(eval_dataset,
validation_metrics)
else:
one_batch = next(iter(eval_dataset_x))
# NOTE: must run predict one batch to initialize parameters
# see: https://www.tensorflow.org/alpha/guide/keras/saving_and_serializing#saving_subclassed_models # noqa: E501
keras_model.predict_on_batch(one_batch)
keras_model.load_weights(save)
result = keras_model.evaluate(eval_dataset)
assert (len(result) == len(validation_metrics) + 1)
result_metrics = dict()
for idx, m in enumerate(["loss"] + validation_metrics):
result_metrics[m] = result[idx]
return result_metrics
def keras_compile(estimator, model_params, metric_names):
# remove optimizer param from model_params and use it when call "compile()"
optimizer = None
loss = None
if "optimizer" in model_params:
optimizer = model_params["optimizer"]
del model_params["optimizer"]
if "loss" in model_params:
loss = model_params["loss"]
del model_params["loss"]
classifier_pkg = sys.modules[estimator.__module__]
model_metrics = []
if hasattr(classifier_pkg, "eval_metrics_fn"):
metrics_functions = classifier_pkg.eval_metrics_fn()
for key, func in metrics_functions.items():
func.__name__ = key
model_metrics.append(func)
# use WITH specified metrics if it's not default.
if metric_names != ["Accuracy"]:
keras_metrics = metrics.get_keras_metrics(metric_names)
else:
if len(model_metrics) > 0:
keras_metrics = model_metrics
else:
keras_metrics = metrics.get_keras_metrics(["Accuracy"])
# setting optimizer
has_none_optimizer = False
if optimizer is None:
# use keras model default optimizer if optimizer is not specified in
# WITH clause.
members = inspect.getmembers(classifier_pkg)
# default optimizer
optimizer = tf.keras.optimizers.Adagrad(lr=0.001)
for m, func in members:
if m == "optimizer":
optimizer = classifier_pkg.optimizer()
if optimizer is None:
has_none_optimizer = True
warnings.warn('optimizer() returns None')
if loss is None:
members = inspect.getmembers(classifier_pkg)
# FIXME(typhoonzero): default loss may cause error if model's output
# shape does not fit.
loss = "sparse_categorical_crossentropy"
for m, func in members:
if m == "loss":
loss = classifier_pkg.loss
classifier = init_model_with_feature_column(
estimator, model_params, has_none_optimizer=has_none_optimizer)
# FIXME(sneaxiy): some models defined by other framework (not TensorFlow or
# XGBoost) may return None optimizer.
# For example:
# https://github.com/sql-machine-learning/models/blob/ce970d14a524e20de10a645c99b6bf8724be17d9/sqlflow_models/arima_with_stl_decomposition.py#L123 # noqa: E501
if has_none_optimizer:
assert hasattr(
classifier,
"sqlflow_train_loop"), "optimizer() should not return None"
else:
classifier.compile(optimizer=optimizer,
loss=loss,
metrics=keras_metrics)
return classifier, has_none_optimizer