def test_save_frozen(tmp_path, model, training_data, feed_dict, output_names):
# Not currently supported for tf2.0
if MAJOR_VERSION == 2:
return
save_path = os.path.join(str(tmp_path), "model_output")
tmp_path = os.path.join(str(tmp_path), "model_output_tmp")
x, y = training_data
orig_out = model.predict(x)
tf_utils.save_model(model,
save_path,
tmp_path=tmp_path,
export_type="tf_frozen")
tf.keras.backend.clear_session()
graph = tf.python.Graph()
sess = tf.python.Session(graph=graph)
with sess.as_default() as default_sess:
with sess.graph.as_default() as default_graph:
graph_def = tf.python.GraphDef()
graph_def.ParseFromString(tf_utils.read_file(save_path, "rb"))
tf.import_graph_def(graph_def, name="", return_elements=None)
loaded_out = sess.run(output_names, feed_dict=feed_dict)
assert np.allclose(orig_out, loaded_out)
def test_save_optimized(
tmp_path,
model,
training_data,
feed_dict,
output_names):
save_path = os.path.join(str(tmp_path), "model_output")
tmp_save_path = os.path.join(str(save_path), "tmp_model_output")
x, y = training_data
orig_out = model.predict(x)
tf_utils.save_model(
model,
save_path,
tmp_path=tmp_save_path,
export_type="tf_optimized")
graph = tf.python.Graph()
sess = tf.python.Session(graph=graph)
with sess.as_default() as default_sess:
with sess.graph.as_default() as default_graph:
graph_def = tf.python.GraphDef()
graph_def.ParseFromString(
tf_utils.read_file(os.path.join(
save_path, "optimized_graph.pb"), "rb"))
tf.import_graph_def(
graph_def, name="", return_elements=None)
loaded_out = sess.run(output_names, feed_dict=feed_dict)
assert np.allclose(orig_out, loaded_out)
def __train_instance(self, instance, x, y, **fit_kwargs):
tf_utils.clear_tf_session()
# Reload the Instance
instance = self.__load_instance(instance)
# Fit the model
if not self.state.dry_run:
instance.fit(x, y, **fit_kwargs)
def _write_result_file(self):
"""Record results - one file per instance"""
status = self._make_status()
status_json = json.dumps(status)
prefix = self._get_filename_prefix(with_execution_info=False)
# don't do a os.join as it is just appending a suffix
fname = prefix + '-results.json'
tf_utils.write_file(fname, status_json)
# send result to the cloud service
if self.cloudservice.is_enable:
self.cloudservice.send_results(status)
def test_save_keras_bundle(tmp_path, model, training_data):
save_path = os.path.join(str(tmp_path), "model_output")
tmp_path = os.path.join(str(tmp_path), "model_output_tmp")
x, y = training_data
tf_utils.save_model(
model, save_path, tmp_path=tmp_path, export_type="keras_bundle")
loaded = tf.keras.models.load_model(save_path + ".keras_bundle.h5")
orig_out = model.predict(x)
loaded_out = loaded.predict(x)
assert np.allclose(orig_out, loaded_out)
def save_best_models(self, export_type="keras", num_models=1):
""" Exports the best model based on the specified metric, to the
results directory.
Args:
output_type (str, optional): Defaults to "keras". What format
of model to export:
# Tensorflow 1.x/2.x
"keras" - Save as separate config (JSON) and weights (HDF5)
files.
"keras_bundle" - Saved in Keras's native format (HDF5), via
save_model()
# Currently only supported in Tensorflow 1.x
"tf" - Saved in tensorflow's SavedModel format. See:
https://www.tensorflow.org/alpha/guide/saved_model
"tf_frozen" - A SavedModel, where the weights are stored
in the model file itself, rather than a variables
directory. See:
https://www.tensorflow.org/guide/extend/model_files
"tf_optimized" - A frozen SavedModel, which has
additionally been transformed via tensorflow's graph
transform library to remove training-specific nodes
and operations. See:
https://github.com/tensorflow/tensorflow/tree/master/tensorflow/tools/graph_transforms
"tf_lite" - A TF Lite model.
"""
instance_states, execution_states, models = self.get_best_models(
num_models=num_models, compile=False)
zipped = zip(models, instance_states, execution_states)
for idx, (model, instance_state, execution_state) in enumerate(zipped):
export_prefix = "%s-%s-%s-%s" % (
self.state.project, self.state.architecture,
instance_state.idx, execution_state.idx)
export_path = os.path.join(self.state.host.export_dir,
export_prefix)
tmp_path = os.path.join(self.state.host.tmp_dir, export_prefix)
info("Exporting top model (%d/%d) - %s" %
(idx + 1, len(models), export_path))
tf_utils.save_model(model,
export_path,
tmp_path=tmp_path,
export_type=export_type)
def model_from_configs(model_config,
loss_config,
optimizer_config,
metrics_config,
weights_filename=None):
"""Creates a Keras model from the configurations typically stored in
an InstanceState.
Args:
model_config (dict): Configuration dictionary representing the
model.
loss_config (dict, list, or str): Configuration representing the
loss(es) for the model.
optimizer_config (dict): Configuration representing the optimizer.
metrics_config (dict, list, or str): Configuration representing the
metrics.
weights_filename (str, optional): Filename containing weights to
load.
Returns:
Model: The Keras Model defined by the config objects.
"""
model = model_from_json(json.dumps(model_config))
loss = tf_utils.deserialize_loss(loss_config)
optimizer = tf.keras.optimizers.deserialize(optimizer_config) # nopep8
metrics = json.loads(metrics_config)
model.compile(loss=loss, optimizer=optimizer, metrics=metrics)
if weights_filename:
if tf.io.gfile.exists(weights_filename):
model.load_weights(weights_filename)
else:
warning("No weights file: '%s'" % weights_filename)
return model
def _report_status_worker(self):
"Report tuner status periodically"
# getting stats
status = self._make_status()
# needed for cloudservice
status['training_complete'] = self.training_complete
status['epoch_history'] = self.epoch_history
status_json = json.dumps(status)
# write on disk
fname = path.join(self.tuner_state.host.results_dir, 'status.json')
tf_utils.write_file(fname, status_json)
# send status to cloudservice
if self.cloudservice.is_enable:
self.cloudservice.send_status(status)
def test_save_tf_lite(
tmp_path,
model,
training_data,
feed_dict,
output_names):
# There are bugs with saving as tf.lite in early version
# see: https://github.com/tensorflow/tensorflow/issues/17349
if MAJOR_VERSION == 1 and MINOR_VERSION <= 13:
return
save_path = os.path.join(str(tmp_path), "model_output")
save_file = save_path + ".tflite"
os.makedirs(save_path)
x, y = training_data
orig_out = model.predict(x)
tf_utils.save_model(model, save_path, tmp_path=tmp_path,
export_type="tf_lite")
with tf.python.Session().as_default() as sess:
with tf.Graph().as_default() as _:
interpreter = tf.lite.Interpreter(model_path=save_file)
interpreter.allocate_tensors()
input_details = interpreter.get_input_details()
output_details = interpreter.get_output_details()
for i in range(len(x["i1"])):
i1 = np.expand_dims(np.expand_dims(
x["i1"][i], axis=-1), axis=-1)
i2 = np.expand_dims(np.expand_dims(
x["i2"][i], axis=-1), axis=-1)
interpreter.set_tensor(
input_details[0]['index'], i1)
interpreter.set_tensor(
input_details[1]['index'], i2)
interpreter.invoke()
o1 = interpreter.get_tensor(output_details[0]['index'])
o2 = interpreter.get_tensor(output_details[1]['index'])
assert np.allclose(orig_out[0][i], o1)
assert np.allclose(orig_out[1][i], o2)
def _checkpoint_model(self):
"""Checkpoint model"""
prefix = self._get_filename_prefix()
base_filename = prefix
tmp_path = path.join(self.tuner_state.host.tmp_dir,
path.basename(prefix))
try:
tf_utils.save_model(self.model,
base_filename,
tmp_path=tmp_path,
export_type="keras")
except:
print("FAILED")
import traceback
traceback.print_exc()
write_log("Failed.")
exit(0)
def test_summary(hparams, basic_model, capsys):
idx = '3713'
state = InstanceState(idx, basic_model, hparams)
state.summary()
captured = capsys.readouterr()
to_test = [
'model size: %s' % tf_utils.compute_model_size(basic_model),
'idx: %s' % idx,
]
for s in to_test:
assert s in captured.out
def test_save_keras(tmp_path, model, training_data):
save_path = os.path.join(str(tmp_path), "model_output")
tmp_path = os.path.join(str(tmp_path), "model_output_tmp")
x, y = training_data
tf_utils.save_model(model,
save_path,
tmp_path=tmp_path,
export_type="keras")
config = tf_utils.read_file(save_path + "-config.json")
loaded = tf.keras.models.model_from_json(config)
loaded.load_weights(save_path + "-weights.h5")
loaded.compile(optimizer="adam", loss={"o1": "mse", "o2": "mse"})
orig_out = model.predict(x)
loaded_out = loaded.predict(x)
assert np.allclose(orig_out, loaded_out)
def test_save_tf(
tmp_path,
model,
training_data,
feed_dict,
output_names):
save_path = os.path.join(str(tmp_path), "model_output")
tmp_path = os.path.join(str(tmp_path), "model_output_tmp")
x, y = training_data
tf_utils.save_model(model, save_path, tmp_path=tmp_path, export_type="tf")
orig_out = model.predict(x)
if MAJOR_VERSION >= 2:
sess = None
loaded_model = tf_utils.load_savedmodel(
sess,
export_dir=save_path,
tags=["serve"])
loaded_out = loaded_model.predict(x)
assert np.allclose(orig_out, loaded_out)
else:
with tf.python.Session().as_default() as sess:
loaded_model = tf_utils.load_savedmodel(
sess,
export_dir=save_path,
tags=["serve"])
node_map = {}
for node in sess.graph.as_graph_def().node:
node_map[node.name] = node
output_node = node_map["o2/BiasAdd"]
output_node.input[0] == "o2/MatMul"
output_node.input[1] == "o2/BiasAdd/ReadVariableOp"
output_node = node_map["o1/BiasAdd"]
output_node.input[0] == "o1/MatMul"
output_node.input[1] == "o1/BiasAdd/ReadVariableOp"
def __init__(self, **kwargs):
super(HostState, self).__init__(**kwargs)
self.results_dir = self._register('results_dir', 'results/', True)
self.tmp_dir = self._register('tmp_dir', 'tmp/')
self.export_dir = self._register('export_dir', 'export/', True)
# ensure the user don't shoot himself in the foot
if self.results_dir == self.tmp_dir:
fatal('Result dir and tmp dir must be different')
# create directory if needed
tf_utils.create_directory(self.results_dir)
tf_utils.create_directory(self.tmp_dir, remove_existing=True)
tf_utils.create_directory(self.export_dir)
# init _HOST
config._Host = Host()
status = config._Host.get_status()
tf_version = status['software']['tensorflow']
if tf_version:
major, minor, rev = tf_version.split('.')
if major == '1':
if int(minor) >= 13:
print('ok')
else:
fatal(
"Keras Tuner only work with TensorFlow version >= 1.13\
current version: %s - please upgrade" % tf_version)
else:
warning('Could not determine TensorFlow version.')
def __init__(self, idx, model, hyper_parameters):
super(InstanceState, self).__init__()
self.start_time = int(time.time())
self.idx = idx
# training info
self.training_size = -1
self.validation_size = -1
self.batch_size = -1
self.execution_trained = 0
self.execution_states_collection = ExecutionStatesCollection()
# model info
# we use deepcopy to avoid mutation due to tuners that swap models
self.model_size = tf_utils.compute_model_size(model)
self.optimizer_config = deepcopy(
tf.keras.optimizers.serialize(model.optimizer)) # nopep8
self.loss_config = deepcopy(tf_utils.serialize_loss(model.loss))
self.model_config = json.loads(model.to_json())
self.metrics_config = self.__serialize_metrics(model.metrics)
self.hyper_parameters = deepcopy(hyper_parameters)
self.agg_metrics = None
self.is_best_model = False
self.objective = None # needed by tools that only have this state
def reload_instance(self, idx, execution="last", metrics=[]):
tf_utils.clear_tf_session()
instance_state = self.instance_states.get(idx)
if not instance_state:
raise ValueError("Attempted to reload unknown instance '%s'." %
idx)
# Find the specified execution.
executions = instance_state.execution_states_collection
execution_state = None
if execution == "last":
execution_state = executions.get_last()
elif execution == "best":
execution_state = executions.sort_by_metric(
self.state.objective).to_list()[0]
elif execution:
execution_state = executions.get(idx)
weights_filename = get_weights_filename(self.state, instance_state,
execution_state)
model = InstanceState.model_from_configs(
instance_state.model_config,
instance_state.loss_config,
instance_state.optimizer_config,
instance_state.metrics_config,
weights_filename=weights_filename)
instance = Instance(idx=instance_state.idx,
model=model,
hparams=instance_state.hyper_parameters,
tuner_state=self.state,
cloudservice=self.cloudservice,
instance_state=instance_state)
return instance
def __init__(self, results_dir, tmp_dir, export_dir):
# caching
self.cached_status = None
self.cache_ts = None
self.cache_ttl = 3
# compute static information once
self.gpu_driver_version = 'N/A'
self.nvidia_smi = self._find_nvidia_smi()
self.cpu_core_count = psutil.cpu_count()
self.cpu_frequency = psutil.cpu_freq()
self.partitions = psutil.disk_partitions()
self.hostname = platform.node()
self.cpu_name = platform.processor()
# additional GPU info
if hasattr(tf, 'test'):
self.tf_can_use_gpu = tf.test.is_gpu_available(
) # before get_software
else:
# Support tensorflow versions where tf.test is unavailable.
self.tf_can_use_gpu = False
self._get_gpu_usage() # to get gpu driver info > before get software
self.software = self._get_software()
self.results_dir = results_dir or 'results'
self.tmp_dir = tmp_dir or 'tmp_dir'
self.export_dir = export_dir or 'export'
# ensure the user don't shoot himself in the foot
if self.results_dir == self.tmp_dir:
fatal('Result dir and tmp dir must be different')
# create directory if needed
tf_utils.create_directory(self.results_dir)
tf_utils.create_directory(self.tmp_dir, remove_existing=True)
tf_utils.create_directory(self.export_dir)
def new_instance(self):
"Return a never seen before model instance"
fail_streak = 0
collision_streak = 0
over_sized_streak = 0
while 1:
# clean-up TF graph from previously stored (defunct) graph
tf_utils.clear_tf_session()
self.stats.generated_instances += 1
fail_streak += 1
try:
model = self.model_fn()
except:
if self.state.debug:
traceback.print_exc()
self.stats.invalid_instances += 1
warning("invalid model %s/%s" %
(self.stats.invalid_instances, self.max_fail_streak))
if self.stats.invalid_instances >= self.max_fail_streak:
warning("too many consecutive failed models - stopping")
return None
continue
# stop if the model_fn() return nothing
if not model:
warning("No model returned from model function - stopping.")
return None
# computing instance unique idx
idx = self.__compute_model_id(model)
if self.instance_states.exist(idx):
collision_streak += 1
self.stats.collisions += 1
warning("Collision for %s -- skipping" % (idx))
if collision_streak >= self.max_fail_streak:
return None
continue
# check size
nump = tf_utils.compute_model_size(model)
if nump > self.state.max_model_parameters:
over_sized_streak += 1
self.stats.over_sized_models += 1
warning("Oversized model: %s parameters-- skipping" % (nump))
if over_sized_streak >= self.max_fail_streak:
warning("too many consecutive failed model - stopping")
return None
continue
# creating instance
hparams = config._DISTRIBUTIONS.get_hyperparameters()
instance = Instance(idx, model, hparams, self.state,
self.cloudservice)
break
# recording instance
self.instance_states.add(idx, instance.state)
return instance