def test_load_saved_model_with_no_variables(self, builder_cls):
"""Test that SavedModel runs saver when there appear to be no variables.
When no variables are detected, this may mean that the variables were saved
to different collections, or the collections weren't saved to the
SavedModel. If the SavedModel MetaGraphDef contains a saver, it should still
run in either of these cases.
Args:
builder_cls: SavedModelBuilder or _SavedModelBuilder class
"""
# Force test to run in graph mode.
# The SavedModelBuilder.add_meta_graph_and_variables and
# SavedModelLoader.load methods are v1-only APIs that require a session to
# work.
with ops.Graph().as_default():
path = _get_export_dir("no_variable_saved_model")
with session.Session(graph=ops.Graph()) as sess:
x = variables.VariableV1(5,
name="x",
collections=["not_global_variable"])
y = variables.VariableV1(11,
name="y",
collections=["not_global_variable"])
self.assertFalse(variables._all_saveable_objects())
z = x + y
self.evaluate(variables.variables_initializer([x, y]))
foo_sig_def = signature_def_utils.build_signature_def(
{"foo_input": utils.build_tensor_info(x)},
{"foo_output": utils.build_tensor_info(z)})
builder = saved_model_builder.SavedModelBuilder(path)
builder.add_meta_graph_and_variables(
sess, ["foo_graph"], {"foo": foo_sig_def},
saver=tf_saver.Saver([x, y]))
builder.save()
loader = loader_impl.SavedModelLoader(path)
with self.session(graph=ops.Graph()) as sess:
saver, _ = loader.load_graph(sess.graph, ["foo_graph"])
self.assertFalse(variables._all_saveable_objects())
self.assertIsNotNone(saver)
with self.session(graph=ops.Graph()) as sess:
loader.load(sess, ["foo_graph"])
self.assertEqual(5, sess.run(_tensor_name("x")))
self.assertEqual(11, sess.run(_tensor_name("y")))
def add_meta_graph(self,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None):
"""Adds the current meta graph to the SavedModel.
Creates a Saver in the current scope and uses the Saver to export the meta
graph def. Invoking this API requires the `add_meta_graph_and_variables()`
API to have been invoked before.
Args:
tags: The set of tags to annotate the meta graph def with.
signature_def_map: The map of signature defs to be added to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel. Note
that this collection should be a subset of the assets saved as part of
the first meta graph in the SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded.
Raises:
AssertionError: If the variables for the SavedModel have not been saved
yet.
"""
if not self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has not been saved yet. "
"Please invoke `add_meta_graph_and_variables()` first.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Save asset files and write them to disk, if any.
self._save_and_write_assets(assets_collection)
if main_op is None:
# Add legacy init op to the SavedModel.
self._maybe_add_legacy_init_op(legacy_init_op)
else:
self._add_main_op(main_op)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
def store(graph_name, target):
saver = tf.train.import_meta_graph(graph_name + ".meta", clear_devices=True)
saver._var_list = variables._all_saveable_objects()
# print(variables._all_saveable_objects())
with open(os.path.join(os.path.dirname(graph_name), "net_io_names.json"), "r") as f:
net_io_names = json.load(f)
with tf.Session() as sess:
saver.restore(sess, graph_name)
# print(variables._all_saveable_objects())
# print(saver._var_list)
builder = tf.saved_model.builder.SavedModelBuilder(target)
input = sess.graph.as_graph_element(net_io_names["raw"])
output = sess.graph.as_graph_element(net_io_names["dist"])
input_info = tf.saved_model.utils.build_tensor_info(input)
output_info = tf.saved_model.utils.build_tensor_info(output)
signature = tf.saved_model.signature_def_utils.build_signature_def(
inputs={"raw": input_info},
outputs={"dist": output_info},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME,
)
builder.add_meta_graph_and_variables(
sess,
[tf.saved_model.tag_constants.SERVING],
signature_def_map={"predict_images": signature},
)
builder.save(as_text=True)
def _get_saveable_variables(exclude_scopes=tuple()):
all_vars = variables._all_saveable_objects()
vars_to_train = [
var for var in all_vars
if all(sc not in var.name for sc in exclude_scopes)
]
return vars_to_train
def restore_variables(self, sess, saver, import_scope=None):
"""Restore SavedModel variable values into the session.
Args:
sess: tf.Session to restore variable values.
saver: a tf.train.Saver object. Can be None if there are no variables in
graph. This may be the saver returned by the load_graph() function, or a
default `tf.train.Saver()`.
import_scope: Optional `string` -- if specified, prepend this string
followed by '/' to all loaded tensor names. This scope is applied to
tensor instances loaded into the passed session, but it is *not* written
through to the static `MetaGraphDef` protocol buffer that is returned.
Raises:
ValueError: if no saver was passed to the saver argument, and there are
variables in the graph.
"""
with sess.graph.as_default():
if (saver is None and
not variables._all_saveable_objects(scope=import_scope)): # pylint: disable=protected-access
tf_logging.info("The specified SavedModel has no variables; no "
"checkpoints were restored.")
elif isinstance(saver, tf_saver.Saver):
saver.restore(sess, self._variables_path)
else:
raise ValueError(
"No tf.train.Saver object was passed to the function "
"SavedModelLoader.restore_variables. Since there are variables in "
"the graph, a saver is required.")
def is_graph_frozen() -> bool:
"""
Checks if graph in current graph is frozen
:return: `True` or `False`
"""
return not bool(variables._all_saveable_objects())
def compile_train_graph(config,hp):
class net:
pass
net.training = tf.placeholder(tf.bool)
net.global_step = tf.get_variable('global_step', trainable=False, shape=(),
dtype=tf.int32,
initializer=tf.zeros_initializer())
net.x = tf.placeholder(tf.float32, shape=[hp.batch_size, hp.sequence_len])
net.seq_length = tf.placeholder(tf.int32, shape=[hp.batch_size])
net.y_indexs = tf.placeholder(tf.int64)
net.y_values = tf.placeholder(tf.int32)
net.y_shape = tf.placeholder(tf.int64)
net.y = tf.SparseTensor(net.y_indexs, net.y_values, net.y_shape)
net.logits, net.ratio = model.inference(net.x, net.seq_length, net.training,hp.sequence_len,configure = config)
if 'fl_gamma' in config.keys():
net.ctc_loss = model.loss(net.logits, net.seq_length, net.y, fl_gamma = config['fl_gamma'])
else:
net.ctc_loss = model.loss(net.logits, net.seq_length, net.y)
net.opt = model.train_opt(hp.step_rate,
hp.max_steps,
global_step=net.global_step,
opt_name = config['opt_method'])
if hp.gradient_clip is None:
net.step = net.opt.minimize(net.ctc_loss,global_step = net.global_step)
else:
net.gradients, net.variables = zip(*net.opt.compute_gradients(net.ctc_loss))
net.gradients = [None if gradient is None else tf.clip_by_norm(gradient, hp.gradient_clip) for gradient in net.gradients]
net.step = net.opt.apply_gradients(zip(net.gradients, net.variables),global_step = net.global_step)
net.error,net.errors,net.y_ = model.prediction(net.logits, net.seq_length, net.y)
net.init = tf.global_variables_initializer()
net.variable_to_restore=set(variables._all_saveable_objects()+tf.moving_average_variables())
net.saver = tf.train.Saver(var_list=net.variable_to_restore,
save_relative_paths=True)
net.summary = tf.summary.merge_all()
return net
def restore_variables(self, sess, saver, import_scope=None):
"""Restore SavedModel variable values into the session.
Args:
sess: tf.Session to restore variable values.
saver: a tf.train.Saver object. Can be None if there are no variables in
graph. This may be the saver returned by the load_graph() function, or a
default `tf.train.Saver()`.
import_scope: Optional `string` -- if specified, prepend this string
followed by '/' to all loaded tensor names. This scope is applied to
tensor instances loaded into the passed session, but it is *not* written
through to the static `MetaGraphDef` protocol buffer that is returned.
Raises:
ValueError: if no saver was passed to the saver argument, and there are
variables in the graph.
"""
with sess.graph.as_default():
if (saver is None and
not variables._all_saveable_objects(scope=import_scope)): # pylint: disable=protected-access
tf_logging.info(
"The specified SavedModel has no variables; no "
"checkpoints were restored.")
elif isinstance(saver, tf_saver.Saver):
saver.restore(sess, self._variables_path)
else:
raise ValueError(
"No tf.train.Saver object was passed to the function "
"SavedModelLoader.restore_variables. Since there are variables in "
"the graph, a saver is required.")
def add_meta_graph(self,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None):
"""Adds the current meta graph to the SavedModel.
Creates a Saver in the current scope and uses the Saver to export the meta
graph def. Invoking this API requires the `add_meta_graph_and_variables()`
API to have been invoked before.
Args:
tags: The set of tags to annotate the meta graph def with.
signature_def_map: The map of signature defs to be added to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel. Note
that this collection should be a subset of the assets saved as part of
the first meta graph in the SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded.
Raises:
AssertionError: If the variables for the SavedModel have not been saved
yet.
"""
if not self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has not been saved yet. "
"Please invoke `add_meta_graph_and_variables()` first.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Save asset files and write them to disk, if any.
self._save_and_write_assets(assets_collection)
if main_op is None:
# Add legacy init op to the SavedModel.
self._maybe_add_legacy_init_op(legacy_init_op)
else:
self._add_main_op(main_op)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
def _get_saveable_variables(exclude_scopes=tuple()):
# noinspection PyProtectedMember
all_vars = variables._all_saveable_objects()
vars_to_train = [
var for var in all_vars
if all(sc not in var.name for sc in exclude_scopes)
]
return vars_to_train
def _is_graph_frozen() -> bool:
"""
Checks if graph in current graph is frozen
:return: `True` or `False`
"""
from tensorflow.python.ops import variables
return not bool(variables._all_saveable_objects())
def __init__(self, FLAGS, id2word, word2id, emb_matrix):
"""
Initializes the QA model.
Inputs:
FLAGS: the flags passed in from main.py
id2word: dictionary mapping word idx (int) to word (string)
word2id: dictionary mapping word (string) to word idx (int)
emb_matrix: numpy array shape (400002, embedding_size) containing pre-traing GloVe embeddings
"""
print "Initializing the QAModel..."
self.FLAGS = FLAGS
self.id2word = id2word
self.word2id = word2id
self.emb = emb_matrix
self.lr = FLAGS.learning_rate
# Add all parts of the graph
with tf.variable_scope(
"QAModel",
initializer=tf.contrib.layers.variance_scaling_initializer(
factor=1.0, uniform=True)):
self.add_placeholders()
self.add_embedding_layer(emb_matrix.shape)
self.build_graph()
self.add_loss()
if self.FLAGS.mode == 'train':
# Define trainable parameters, gradient, gradient norm, and clip by gradient norm
params = tf.trainable_variables()
gradients = tf.gradients(self.loss, params)
self.gradient_norm = tf.global_norm(gradients)
clipped_gradients, _ = tf.clip_by_global_norm(
gradients, FLAGS.max_gradient_norm)
self.param_norm = tf.global_norm(params)
# self.loss += self.param_norm * 0.03
# Define optimizer and updates
# (updates is what you need to fetch in session.run to do a gradient update)
self.global_step = tf.Variable(0,
name="global_step",
trainable=False)
# Exponential decay
#starter_learning_rate = FLAGS.learning_rate
#learning_rate = tf.train.exponential_decay(starter_learning_rate, self.global_step, 7000, 0.5, staircase=True)
opt = tf.train.AdamOptimizer(learning_rate=self.learning_rate
) # you can try other optimizers
self.updates = opt.apply_gradients(zip(clipped_gradients, params),
global_step=self.global_step)
# Define savers (for checkpointing) and summaries (for tensorboard)
self.saver = tf.train.Saver(variables._all_saveable_objects()[1:],
max_to_keep=FLAGS.keep)
self.bestmodel_saver = tf.train.Saver(max_to_keep=1)
self.summaries = tf.summary.merge_all()
def get_variable_dict():
"""Returns a dictionary of all existing tensorflow variables.
Returns:
A dictionary mapping the variable name to the variable.
"""
var_list = variables._all_saveable_objects()
var_list.append(tf.train.get_or_create_global_step())
var_dict = {var.op.name: var for var in var_list}
return var_dict
def _maybe_create_saver(self, saver=None):
"""Creates a sharded saver if one does not already exist."""
if not saver:
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
return saver
def serialize_graph(clear_devices=False, as_text=False):
"""serialize tf graph to path."""
saver = tf_saver.Saver(variables._all_saveable_objects(),
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices)
if as_text:
return str(meta_graph_def)
else:
return meta_graph_def.SerializeToString()
def _maybe_create_saver(self, saver=None):
"""Creates a sharded saver if one does not already exist."""
if not saver:
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
return saver
def test_load_saved_model_with_no_variables(self, builder_cls):
"""Test that SavedModel runs saver when there appear to be no variables.
When no variables are detected, this may mean that the variables were saved
to different collections, or the collections weren't saved to the
SavedModel. If the SavedModel MetaGraphDef contains a saver, it should still
run in either of these cases.
Args:
builder_cls: SavedModelBuilder or _SavedModelBuilder class
"""
path = _get_export_dir("no_variable_saved_model")
with session.Session(graph=ops.Graph()) as sess:
x = variables.VariableV1(
5, name="x", collections=["not_global_variable"])
y = variables.VariableV1(
11, name="y", collections=["not_global_variable"])
self.assertFalse(variables._all_saveable_objects())
z = x + y
self.evaluate(variables.variables_initializer([x, y]))
foo_sig_def = signature_def_utils.build_signature_def(
{"foo_input": utils.build_tensor_info(x)},
{"foo_output": utils.build_tensor_info(z)})
builder = saved_model_builder.SavedModelBuilder(path)
builder.add_meta_graph_and_variables(
sess, ["foo_graph"], {"foo": foo_sig_def},
saver=tf_saver.Saver([x, y]))
builder.save()
loader = loader_impl.SavedModelLoader(path)
with self.session(graph=ops.Graph()) as sess:
saver, _ = loader.load_graph(sess.graph, ["foo_graph"])
self.assertFalse(variables._all_saveable_objects())
self.assertIsNotNone(saver)
with self.session(graph=ops.Graph()) as sess:
loader.load(sess, ["foo_graph"])
self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
self.assertEqual(11, sess.graph.get_tensor_by_name("y:0").eval())
def get_tf_graph(clear_devices=False, model_scope=''):
global _TF_GRAPH_DICT
cur_graph = _TF_GRAPH_DICT[model_scope]
if cur_graph is None:
raise ValueError('get tf graph under model scope[%s] failed!' %
model_scope)
with cur_graph.as_default():
saver = tf_saver.Saver(variables._all_saveable_objects(),
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
return saver.export_meta_graph(clear_devices=clear_devices)
def _build(self, checkpoint_path, build_save, build_restore):
"""Builds saver_def."""
if not context.executing_eagerly():
if self._is_built:
return
self._is_built = True
if not self.saver_def or context.executing_eagerly():
if self._builder is None:
# Attention: this is our target!!
self._builder = SecureBulkSaverBuilder(self._write_version)
if self._var_list is None:
# pylint: disable=protected-access
self._var_list = variables._all_saveable_objects()
if not self._var_list:
if self._allow_empty:
self._is_empty = True
return
else:
raise ValueError("No variables to save")
self._is_empty = False
self.saver_def = self._builder._build_internal( # pylint: disable=protected-access
self._var_list,
reshape=self._reshape,
sharded=self._sharded,
max_to_keep=self._max_to_keep,
keep_checkpoint_every_n_hours=self.
_keep_checkpoint_every_n_hours,
name=self._name,
restore_sequentially=self._restore_sequentially,
filename=checkpoint_path,
build_save=build_save,
build_restore=build_restore)
elif self.saver_def and self._name:
# Since self._name is used as a name_scope by builder(), we are
# overloading the use of this field to represent the "import_scope" as
# well.
self.saver_def.filename_tensor_name = ops.prepend_name_scope(
self.saver_def.filename_tensor_name, self._name)
self.saver_def.save_tensor_name = ops.prepend_name_scope(
self.saver_def.save_tensor_name, self._name)
self.saver_def.restore_op_name = ops.prepend_name_scope(
self.saver_def.restore_op_name, self._name)
self._check_saver_def()
if not context.executing_eagerly():
# Updates next checkpoint time.
# Set in __init__ when executing eagerly.
self._next_checkpoint_time = (
time.time() +
self.saver_def.keep_checkpoint_every_n_hours * 3600)
def test_load_saved_model_with_no_variables(self):
"""Test that SavedModel runs saver when there appear to be no variables.
When no variables are detected, this may mean that the variables were saved
to different collections, or the collections weren't saved to the
SavedModel. If the SavedModel MetaGraphDef contains a saver, it should still
run in either of these cases.
"""
path = _get_export_dir("no_variable_saved_model")
with session.Session(graph=ops.Graph()) as sess:
x = variables.Variable(5,
name="x",
collections=["not_global_variable"])
y = variables.Variable(11,
name="y",
collections=["not_global_variable"])
self.assertFalse(variables._all_saveable_objects())
z = x + y
sess.run(variables.variables_initializer([x, y]))
foo_sig_def = signature_def_utils.build_signature_def(
{"foo_input": utils.build_tensor_info(x)},
{"foo_output": utils.build_tensor_info(z)})
builder = saved_model_builder.SavedModelBuilder(path)
builder.add_meta_graph_and_variables(sess, ["foo_graph"],
{"foo": foo_sig_def},
saver=tf_saver.Saver([x, y]))
builder.save()
loader = loader_impl.SavedModelLoader(path)
with self.test_session(graph=ops.Graph()) as sess:
saver = loader.load_graph(sess.graph, ["foo_graph"])
self.assertFalse(variables._all_saveable_objects())
self.assertIsNotNone(saver)
with self.test_session(graph=ops.Graph()) as sess:
loader.load(sess, ["foo_graph"])
self.assertEqual(5, sess.graph.get_tensor_by_name("x:0").eval())
self.assertEqual(11, sess.graph.get_tensor_by_name("y:0").eval())
def serialize_graph(clear_devices=False, as_text=False, model_scope=''):
"""serialize tf graph to path."""
global _TF_GRAPH_DICT
cur_graph = _TF_GRAPH_DICT[model_scope]
if cur_graph is None:
raise ValueError('get tf graph under model scope[%s] failed!' %
model_scope)
with cur_graph.as_default():
saver = tf_saver.Saver(variables._all_saveable_objects(),
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices)
if as_text:
return str(meta_graph_def)
else:
return meta_graph_def.SerializeToString()
def setup_training(model, batcher):
"""Does setup before starting training (run_training)"""
train_dir = os.path.join(FLAGS.log_root, "train")
if not os.path.exists(train_dir): os.makedirs(train_dir)
default_device = tf.device('/%s'%FLAGS.gpu)
with default_device:
model.build_graph() # build the graph
if FLAGS.convert_to_coverage_model:
assert FLAGS.coverage, "To convert your non-coverage model to a coverage model, run with convert_to_coverage_model=True and coverage=True"
convert_to_coverage_model()
saver = tf.train.Saver(max_to_keep=1) # only keep 1 checkpoint at a time
if FLAGS.pre_path is not None:
# https://www.tensorflow.org/programmers_guide/supervisor
# remove variables that belong to optimization
var_list = variables._all_saveable_objects()
var_list = [v for v in var_list
if not any(v.op.name.endswith(s)
for s in ['Momentum',
'YF_lr', 'YF_lr_factor', 'YF_mu', "YF_clip_thresh"])] # eg "seq2seq/embedding/embedding/Momentum"
pre_train_saver = tf.train.Saver(var_list)
def load_pretrain(sess):
pre_train_saver.restore(sess, FLAGS.pre_path)
else:
load_pretrain = None
sv = tf.train.Supervisor(logdir=train_dir,
is_chief=True,
saver=saver,
summary_op=None,
save_summaries_secs=60, # save summaries for tensorboard every 60 secs
save_model_secs=60, # checkpoint every 60 secs
global_step=model.global_step,
init_fn=load_pretrain)
summary_writer = sv.summary_writer
tf.logging.info("Preparing or waiting for session...")
sess_context_manager = sv.prepare_or_wait_for_session(config=util.get_config())
tf.logging.info("Created session.")
try:
run_training(model, batcher, sess_context_manager, sv, summary_writer) # this is an infinite loop until interrupted
except KeyboardInterrupt:
tf.logging.info("Caught keyboard interrupt on worker. Stopping supervisor...")
sv.stop()
def main():
# define model, optimizer, training, hyperparams
args = config()
_BASEDIR = args.basedirectory
# print('*'*20)
# print(args)
# print('*'*20)
print("Name of the Experiment", args.name_of_experiment, '\n')
# load data
(x_train, y_train), (x_test, y_test) = mnist.load_data()
# reshape from rank 3 tensor to rank 4 tensor
x_train = np.reshape(x_train, (x_train.shape[0], args.image_size[0],
args.image_size[1], args.image_size[2]))
x_test = np.reshape(x_test, (x_test.shape[0], args.image_size[0],
args.image_size[1], args.image_size[2]))
x_train = x_train.astype('float32')
x_test = x_test.astype('float32')
print('Dataset Statistics')
print('Training data shape', x_train.shape)
print('Testing data shape', x_test.shape)
# data generator
datagen = ImageDataGenerator(rescale=1. / 255)
print('Number of training samples', x_train.shape[0])
print('Number of test samples', x_test.shape[0], '\n')
# convert class vectors to binary class matrices
y_train = keras.utils.to_categorical(y_train, args.num_classes)
y_test = keras.utils.to_categorical(y_test, args.num_classes)
x_placeholder = tf.compat.v1.placeholder(
tf.float32,
[None, args.image_size[0], args.image_size[1], args.image_size[2]])
learning_rate = tf.compat.v1.placeholder(tf.float32, name='lr')
# create the model layers
# try:
# with tf.Session(graph=tf.Graph()) as sess:
# tf.saved_model.loader.load(
# sess, tags=["train"], export_dir=model_path)
# except:
model = L1(args, x_placeholder)
model.compile()
print('Compiled Model')
model.print_model()
# y_placeholder_dict = {}
# for op_key, op_val in model._output_layers.items():
# y_placeholder_dict[op_key] = tf.placeholder(
# tf.float32, shape=(op_val['hp'].shape))
y_placeholder = tf.compat.v1.placeholder(tf.float32, shape=(None, 10))
# loss functions
# High precision Cross entropy loss
# print('saveable obj', variables._all_saveable_objects())
# print('local vars', variables.local_variables())
# print(' global vars', variables.global_variables())
# print('model vars', variables.model_variables())
# print('trainable varts', variables.trainable_variables())
# print('moving average vars', variables.moving_average_variables())
hp_cross_entropy_dict = model.hp_cross_entropy(y_placeholder)
hp_cross_entropy = tf.reduce_sum(
input_tensor=list(hp_cross_entropy_dict.values()))
distillation_loss_dict = model.distillation_loss()
distillation_loss = tf.reduce_sum(
input_tensor=list(distillation_loss_dict.values()))
bit_loss = model.bit_loss()
regularization_loss = model.L2_regularization()
# regularization parameter
regularization_term = 0
for k in model._layers_objects.keys():
if isinstance(model._layers_objects[k]['layer_obj'],
gtcConv2d) or isinstance(
model._layers_objects[k]['layer_obj'], gtcDense):
regularization_term = regularization_term + tf.nn.l2_loss(
model._layers_objects[k]['layer_obj'].hp_weights)
# print('---REGULARIZATION', regularization_term)
lambda_distillation_loss = tf.constant(args.lambda_distillation_loss)
lambda_bit_loss = tf.constant(args.lambda_bit_loss)
lambda_regularization = tf.constant(args.lambda_regularization)
# Total loss
losses = {}
lambda_vars = {}
lambda_vars['lambda_bit_loss'] = lambda_bit_loss
lambda_vars['lambda_distillation_loss'] = lambda_distillation_loss
lambda_vars['lambda_regularization'] = lambda_regularization
# optimizer
losses['total_loss'] = hp_cross_entropy + distillation_loss * lambda_vars[
'lambda_distillation_loss'] + bit_loss * lambda_vars[
'lambda_bit_loss'] + lambda_vars[
'lambda_regularization'] * regularization_loss
optimizer = tf.compat.v1.train.AdamOptimizer(learning_rate).minimize(
losses['total_loss'])
metrices = {}
model.lp_accuracy(y_placeholder)
model.hp_accuracy(y_placeholder)
model.compile_training_info(loss=losses,
optimizer=optimizer,
metrices=metrices)
# print('check inputs to network')
# [print(k, v) for k, v in model._layers_strings.items()]
# print('check forward inputs')
# [print(k, v) for k, v in model._forward_prop_inputs.items()]
# print('check layer objects')
# [print(k, v) for k, v in model._layers_objects.items()]
# summary
summary_ops = model.get_summary()
for k, v in summary_ops.items():
if isinstance(v, dict):
for k1, v1 in v.items():
# print(k, k1, v1, '\n')
tf.compat.v1.summary.scalar('_'.join([k, k1]), v1)
else:
# print(k, v, '\n')
tf.compat.v1.summary.scalar('_'.join(k), v)
# print('SUMMARY')
# [print(k, v) for k, v in summary_ops.items()]
validation_log_path = _BASEDIR + '/' + args.name_of_experiment + '/' + 'logs/' + 'validation'
training_log_path = _BASEDIR + '/' + args.name_of_experiment + '/' + 'logs/' + 'train'
model_path = _BASEDIR + '/' + args.name_of_experiment + '/' + 'model/' + 'model.ckpt'
lp_model_path = _BASEDIR + '/' + args.name_of_experiment + '/' + 'model/'
init = tf.compat.v1.global_variables_initializer()
# saving lp model
merge = tf.compat.v1.summary.merge_all()
var_list = variables._all_saveable_objects(
) + model._layers_objects['conv2d']['layer_obj']._save_params()
# print('variable list', var_list)
# saver = tf.train.Saver(var_list=var_list)
sys.stdout.flush()
with tf.compat.v1.Session() as sess:
sess.run(init)
train_writer = tf.compat.v1.summary.FileWriter(training_log_path,
sess.graph)
validation_writer = tf.compat.v1.summary.FileWriter(
validation_log_path, sess.graph)
test_batches = 1
batches = 1
# igraphdef = tf.saved_model.loader.load(
# sess, tags=["train"], export_dir=model_path)
for e in range(args.num_epochs):
test_average = {}
test_average['lp_accuracy', 'flatten_1'] = []
test_average['hp_accuracy', 'flatten_1'] = []
test_average['total_loss', 'total_loss'] = []
test_average['bit_loss'] = []
test_average['distillation_loss', 'flatten_1'] = []
test_average['hp_cross_entropy', 'flatten_1'] = []
test_average['regularization_term'] = []
for x_batch, y_batch in datagen.flow(x_train,
y_train,
batch_size=args.batch_size):
data_dict = {
x_placeholder: x_batch,
y_placeholder: y_batch,
learning_rate: args.learning_rate
}
model.train_on_batch(data_dict)
if batches % 100 == 0:
print('------------ Epoch', e + 1, '/', args.num_epochs,
'batch', batches,
' ---------------------------------')
print_summary_short(model, data_dict)
param_summary = merge.eval(data_dict)
train_writer.add_summary(param_summary, batches)
batches = batches + 1
if batches % 5000 == 0:
args.learning_rate /= 2.
if batches / ((e + 1) * (len(x_train) / args.batch_size)) > 1:
sys.stdout.flush()
# checkpoint.save(file_prefix=checkpoint_prefix)
# saver.save(sess, model_path)
print('Saving the Model')
model._save_model(path=lp_model_path + 'training/',
int_model=False)
for x_batch, y_batch in datagen.flow(
x_test, y_test, batch_size=args.batch_size):
data_dict = {
x_placeholder: x_batch,
y_placeholder: y_batch
}
param_summary = merge.eval(data_dict)
validation_writer.add_summary(param_summary,
test_batches)
test_batches = test_batches + 1
for k1 in test_average.keys():
if isinstance(k1, tuple):
test_average[k1].append(
summary_ops[k1[0]][k1[1]].eval(data_dict))
else:
test_average[k1].append(
summary_ops[k1].eval(data_dict))
if test_batches / (
(e + 1) * (len(x_test) / args.batch_size)) > 1:
print('*' * 20)
print("Test results after epoches", e)
sys.stdout.flush()
[
print(k,
sum(v) / len(v))
for k, v in test_average.items()
]
break
break
if args.print_layers_bits:
print(
'----------------- Number of bits used per each layer ------------------------'
)
bits_per_layer = model._bits_per_layer()
for k, lb in bits_per_layer.items():
if ('conv' in k) or ('dense' in k):
print('k = ', k, ' lb = ', lb.eval())
path_expt = os.path.join(
_BASEDIR, args.name_of_experiment + '/training_model_final/')
if os.path.exists(path_expt):
os.rmdir(path_expt)
os.makedirs(path_expt)
model.save(path=path_expt, int_model=False)
with open(os.path.join(path_expt, 'model_schema.json'),
'w') as schema_f:
schema_f.write(
json.dumps(
{
"dataset": "custom",
"input_names": "Placeholder",
"input_shapes": "1,28,28,1",
"output_names": "Softmax",
"preprocessor": "rgbtogray",
"task": "classifier"
},
indent=4))
int_path_expt = os.path.join(
_BASEDIR, args.name_of_experiment + '/int_model_final/')
if os.path.exists(int_path_expt):
os.rmdir(int_path_expt)
os.makedirs(int_path_expt)
model.save(path=int_path_expt, int_model=True)
with open(os.path.join(int_path_expt, 'model_schema.json'),
'w') as schema_f:
schema_f.write(
json.dumps(
{
"dataset": "custom",
"input_names": "Placeholder",
"input_shapes": "1,28,28,1",
"output_names": "Softmax",
"preprocessor": "rgbtogray",
"task": "classifier"
},
indent=4))
def export_savedmodel(self,
export_dir_base,
serving_input_receiver_fn,
assets_extra=None,
as_text=False,
checkpoint_path=None):
"""Exports inference graph as a SavedModel into given dir.
This method builds a new graph by first calling the
serving_input_receiver_fn to obtain feature `Tensor`s, and then calling
this `Estimator`'s model_fn to generate the model graph based on those
features. It restores the given checkpoint (or, lacking that, the most
recent checkpoint) into this graph in a fresh session. Finally it creates
a timestamped export directory below the given export_dir_base, and writes
a `SavedModel` into it containing a single `MetaGraphDef` saved from this
session.
The exported `MetaGraphDef` will provide one `SignatureDef` for each
element of the export_outputs dict returned from the model_fn, named using
the same keys. One of these keys is always
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY, indicating which
signature will be served when a serving request does not specify one.
For each signature, the outputs are provided by the corresponding
`ExportOutput`s, and the inputs are always the input receivers provided by
the serving_input_receiver_fn.
Extra assets may be written into the SavedModel via the extra_assets
argument. This should be a dict, where each key gives a destination path
(including the filename) relative to the assets.extra directory. The
corresponding value gives the full path of the source file to be copied.
For example, the simple case of copying a single file without renaming it
is specified as `{'my_asset_file.txt': '/path/to/my_asset_file.txt'}`.
Args:
export_dir_base: A string containing a directory in which to create
timestamped subdirectories containing exported SavedModels.
serving_input_receiver_fn: A function that takes no argument and
returns a `ServingInputReceiver`.
assets_extra: A dict specifying how to populate the assets.extra directory
within the exported SavedModel, or `None` if no extra assets are needed.
as_text: whether to write the SavedModel proto in text format.
checkpoint_path: The checkpoint path to export. If `None` (the default),
the most recent checkpoint found within the model directory is chosen.
Returns:
The string path to the exported directory.
Raises:
ValueError: if no serving_input_receiver_fn is provided, no export_outputs
are provided, or no checkpoint can be found.
"""
if serving_input_receiver_fn is None:
raise ValueError('serving_input_receiver_fn must be defined.')
with ops.Graph().as_default() as g:
training.create_global_step(g)
serving_input_receiver = serving_input_receiver_fn()
# Call the model_fn and collect the export_outputs.
estimator_spec = self._call_model_fn(
features=serving_input_receiver.features,
labels=None,
mode=model_fn_lib.ModeKeys.PREDICT)
# Build the SignatureDefs from receivers and all outputs
signature_def_map = build_all_signature_defs(
serving_input_receiver.receiver_tensors,
estimator_spec.export_outputs)
if not checkpoint_path:
# Locate the latest checkpoint
checkpoint_path = saver.latest_checkpoint(self._model_dir)
if not checkpoint_path:
raise ValueError("Couldn't find trained model at %s." %
self._model_dir)
export_dir = get_timestamped_export_dir(export_dir_base)
# TODO(soergel): Consider whether MonitoredSession makes sense here
with tf_session.Session() as session:
saver_for_restore = estimator_spec.scaffold.saver or saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True)
saver_for_restore.restore(session, checkpoint_path)
# TODO(b/36111876): replace legacy_init_op with main_op mechanism
# pylint: disable=protected-access
local_init_op = (
estimator_spec.scaffold.local_init_op
or monitored_session.Scaffold._default_local_init_op())
# pylint: enable=protected-access
# Perform the export
builder = saved_model_builder.SavedModelBuilder(export_dir)
builder.add_meta_graph_and_variables(
session, [tag_constants.SERVING],
signature_def_map=signature_def_map,
assets_collection=ops.get_collection(
ops.GraphKeys.ASSET_FILEPATHS),
legacy_init_op=local_init_op)
builder.save(as_text)
# Add the extra assets
if assets_extra:
assets_extra_path = os.path.join(
compat.as_bytes(export_dir),
compat.as_bytes('assets.extra'))
for dest_relative, source in assets_extra.items():
dest_absolute = os.path.join(
compat.as_bytes(assets_extra_path),
compat.as_bytes(dest_relative))
dest_path = os.path.dirname(dest_absolute)
gfile.MakeDirs(dest_path)
gfile.Copy(source, dest_absolute)
return export_dir
def add_meta_graph_and_variables(self,
sess,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None):
"""Adds the current meta graph to the SavedModel and saves variables.
Creates a Saver to save the variables from the provided session. Exports the
corresponding meta graph def. This function assumes that the variables to be
saved have been initialized. For a given `SavedModelBuilder`, this API must
be called exactly once and for the first meta graph to save. For subsequent
meta graph defs to be added, the `add_meta_graph()` API must be used.
Args:
sess: The TensorFlow session from which to save the meta graph and
variables.
tags: The set of tags with which to save the meta graph.
signature_def_map: The map of signature def map to add to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded.
"""
if self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has "
"already been saved. Please invoke "
"`add_meta_graph()` instead.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Save asset files and write them to disk, if any.
self._save_and_write_assets(assets_collection)
# Create the variables sub-directory, if it does not exist.
variables_dir = os.path.join(
compat.as_text(self._export_dir),
compat.as_text(constants.VARIABLES_DIRECTORY))
if not file_io.file_exists(variables_dir):
file_io.recursive_create_dir(variables_dir)
variables_path = os.path.join(
compat.as_text(variables_dir),
compat.as_text(constants.VARIABLES_FILENAME))
if main_op is None:
# Add legacy init op to the SavedModel.
self._maybe_add_legacy_init_op(legacy_init_op)
else:
self._add_main_op(main_op)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# Save the variables. Also, disable writing the checkpoint state proto. The
# file is not used during SavedModel loading. In addition, since a
# SavedModel can be copied or moved, this avoids the checkpoint state to
# become outdated.
saver.save(sess,
variables_path,
write_meta_graph=False,
write_state=False)
# Export the meta graph def.
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. In the context of the SavedModel, this
# new Saver is the only one that needs to be retained. The associated
# checkpoint that was saved just above contains all of the variable values.
# Thus, any preexisting Savers are redundant and useless at best, but worse
# may break downstream graph-processing tools, and can be confusing during
# debugging. It is therefore safe and wise to set `clear_extraneous_savers`
# to `True`, since it removes both the extraneous SaverDefs and their
# associated Save/Restore Ops from the graph.
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices,
clear_extraneous_savers=True)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
# Mark this instance of SavedModel as having saved variables, such that
# subsequent attempts to save variables will fail.
self._has_saved_variables = True
def add_meta_graph_and_variables(self,
sess,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None):
"""Adds the current meta graph to the SavedModel and saves variables.
Creates a Saver to save the variables from the provided session. Exports the
corresponding meta graph def. This function assumes that the variables to be
saved have been initialized. For a given `SavedModelBuilder`, this API must
be called exactly once and for the first meta graph to save. For subsequent
meta graph defs to be added, the `add_meta_graph()` API must be used.
Args:
sess: The TensorFlow session from which to save the meta graph and
variables.
tags: The set of tags with which to save the meta graph.
signature_def_map: The map of signature def map to add to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded. Note
that when the main_op is specified it is run after the restore op at
load-time.
"""
if self._has_saved_variables:
raise AssertionError("Graph state including variables and assets has "
"already been saved. Please invoke "
"`add_meta_graph()` instead.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Save asset files and write them to disk, if any.
self._save_and_write_assets(assets_collection)
# Create the variables sub-directory, if it does not exist.
variables_dir = os.path.join(
compat.as_text(self._export_dir),
compat.as_text(constants.VARIABLES_DIRECTORY))
if not file_io.file_exists(variables_dir):
file_io.recursive_create_dir(variables_dir)
variables_path = os.path.join(
compat.as_text(variables_dir),
compat.as_text(constants.VARIABLES_FILENAME))
if main_op is None:
# Add legacy init op to the SavedModel.
self._maybe_add_legacy_init_op(legacy_init_op)
else:
self._add_main_op(main_op)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# Save the variables. Also, disable writing the checkpoint state proto. The
# file is not used during SavedModel loading. In addition, since a
# SavedModel can be copied or moved, this avoids the checkpoint state to
# become outdated.
saver.save(sess, variables_path, write_meta_graph=False, write_state=False)
# Export the meta graph def.
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. Removing the preexisting ones was the
# motivation for the clear_extraneous_savers option, but it turns out that
# there are edge cases where that option breaks the graph. Until that is
# resolved, we just leave the option set to False for now.
# TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
# Mark this instance of SavedModel as having saved variables, such that
# subsequent attempts to save variables will fail.
self._has_saved_variables = True
def add_meta_graph_and_variables(self,
sess,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None,
strip_default_attrs=False):
# pylint: disable=line-too-long
"""Adds the current meta graph to the SavedModel and saves variables.
Creates a Saver to save the variables from the provided session. Exports the
corresponding meta graph def. This function assumes that the variables to be
saved have been initialized. For a given `SavedModelBuilder`, this API must
be called exactly once and for the first meta graph to save. For subsequent
meta graph defs to be added, the `add_meta_graph()` API must be used.
Args:
sess: The TensorFlow session from which to save the meta graph and
variables.
tags: The set of tags with which to save the meta graph.
signature_def_map: The map of signature def map to add to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded. Note
that when the main_op is specified it is run after the restore op at
load-time.
strip_default_attrs: Boolean. If `True`, default-valued attributes will be
removed from the NodeDefs. For a detailed guide, see
[Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes).
"""
# pylint: enable=line-too-long
if self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has "
"already been saved. Please invoke "
"`add_meta_graph()` instead.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Add assets and ops
self._add_collections(assets_collection, legacy_init_op, main_op, None)
# Create the variables sub-directory, if it does not exist.
variables_dir = os.path.join(
compat.as_text(self._export_dir),
compat.as_text(constants.VARIABLES_DIRECTORY))
if not file_io.file_exists(variables_dir):
file_io.recursive_create_dir(variables_dir)
variables_path = os.path.join(
compat.as_text(variables_dir),
compat.as_text(constants.VARIABLES_FILENAME))
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# Save the variables. Also, disable writing the checkpoint state proto. The
# file is not used during SavedModel loading. In addition, since a
# SavedModel can be copied or moved, this avoids the checkpoint state to
# become outdated.
saver.save(sess,
variables_path,
write_meta_graph=False,
write_state=False)
# Export the meta graph def.
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. Removing the preexisting ones was the
# motivation for the clear_extraneous_savers option, but it turns out that
# there are edge cases where that option breaks the graph. Until that is
# resolved, we just leave the option set to False for now.
# TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
meta_graph_def = saver.export_meta_graph(
clear_devices=clear_devices,
strip_default_attrs=strip_default_attrs)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
# Mark this instance of SavedModel as having saved variables, such that
# subsequent attempts to save variables will fail.
self._has_saved_variables = True
def add_meta_graph(self,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None):
"""Adds the current meta graph to the SavedModel.
Creates a Saver in the current scope and uses the Saver to export the meta
graph def. Invoking this API requires the `add_meta_graph_and_variables()`
API to have been invoked before.
Args:
tags: The set of tags to annotate the meta graph def with.
signature_def_map: The map of signature defs to be added to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel. Note
that this collection should be a subset of the assets saved as part of
the first meta graph in the SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded.
Raises:
AssertionError: If the variables for the SavedModel have not been saved
yet.
"""
if not self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has not been saved yet. "
"Please invoke `add_meta_graph_and_variables()` first.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Save asset files and write them to disk, if any.
self._save_and_write_assets(assets_collection)
if main_op is None:
# Add legacy init op to the SavedModel.
self._maybe_add_legacy_init_op(legacy_init_op)
else:
self._add_main_op(main_op)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. Removing the preexisting ones was the
# motivation for the clear_extraneous_savers option, but it turns out that
# there are edge cases where that option breaks the graph. Until that is
# resolved, we just leave the option set to False for now.
# TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
def train_from_params(
save_params,
model_params,
train_params,
loss_params=None,
learning_rate_params=None,
optimizer_params=None,
validation_params=None,
load_params=None,
log_device_placement=DEFAULT_PARAMS[
'log_device_placement'], # advanced
dont_run=DEFAULT_PARAMS['dont_run'], # advanced
skip_check=DEFAULT_PARAMS['skip_check'], # advanced
):
"""
Main training interface function.
Args:
save_params (dict):
Describing the parameters used to construct the save database, and
control saving. These include:
- host (str)
Hostname where database connection lives
- port (int)
Port where database connection lives
- dbname (str)
Name of database for storage
- collname (str)
Name of collection for storage
- exp_id (str)
Experiment id descriptor
NOTE: the variables host/port/dbname/coll/exp_id control
the location of the saved data for the run, in order of
increasing specificity. When choosing these, note that:
- If a given host/port/dbname/coll/exp_id already has saved checkpoints,\
then any new call to start training with these same location variables\
will start to train from the most recent saved checkpoint. If you mistakenly\
try to start training a new model with different variable names, or structure,\
from that existing checkpoint, an error will be raised, as the model will be\
incompatiable with the saved variables.
- When choosing what dbname, coll, and exp_id, to use, keep in mind that mongodb\
queries only operate over a single collection. So if you want to analyze\
results from a bunch of experiments together using mongod queries, you should\
put them all in the same collection, but with different exp_ids. If, on the\
other hand, you never expect to analyze data from two experiments together,\
you can put them in different collections or different databases. Choosing\
between putting two experiments in two collections in the same database\
or in two totally different databases will depend on how you want to organize\
your results and is really a matter of preference.
- do_save (bool, default: True)
Whether to save to database
- save_initial_filters (bool, default: True)
Whether to save initial model filters at step = 0,
- save_metrics_freq (int, default: 5)
How often to store train results to database
- save_valid_freq (int, default: 3000)
How often to calculate and store validation results
to database
- save_filters_freq (int, default: 30000)
How often to save filter values to database
- cache_filters_freq (int, default: 3000)
How often to cache filter values locally and save
to ___RECENT database
- cache_max_num (int, default: 6)
Maximal number of cached filters to keep in __RECENT database
- cache_dir (str, default: None)
Path where caches will be saved locally. If None, will default to
~/.tfutils/<host:post>/<dbname>/<collname>/<exp_id>.
model_params (dict): Containing function that produces model and arguments to that function.
- model_params['func']
The function producing the model.
The function's signature is:
Args:
- ``inputs``: data object
- ``train`` (boolean): if in training or testing
- ``seed`` (int): seed for use in random generation
Returns:
- ``outputs`` (tf.Operations): train output tensorflow nodes
- Additional configurations you want to store in database
- Remaining items in model_params are dictionary of arguments passed to func.
train_params (dict): Containing params for data sources and targets in training.
- train_params['data_params']
This contains params for the data
- ``train_params['data_params']['func']`` is the function that constructs the data:
The function's signature is:
Args:
- ``batch_size``: Batch size for input data
Returns:
- ``inputs``: A dictionary of tensors that will be sent to model function
- ``train_params['data_params']['batch_size']`` batch size of the data, will be sent to func
- Remainder of ``train_params['data_params']`` are kwargs passed to func
- train_params['targets'] (optional)
contains params for additional train targets
- ``train_params['targets']['func']`` is a function that produces tensorflow nodes as training targets:
The function's signature is:
Args:
- ``inputs``: returned values of ``train_params['data_params']['func']``
- ``output``: first returned value of ``train_params['model_params']['func']``
Returns:
A dictionary of tensors that will be computed and stored in the database
- Remainder of ``train_parms['targets']`` are arguments to func.
- train_params['validate_first'] (optional, bool, default is True):
controls whether validating before training
- train_params['thres_loss'] (optional, float, default: 100):
If loss exceeds this during training, HiLossError is thrown
- train_params['num_steps'] (int or None, default: None):
How many total steps of the optimization are run.
If None, train is run until process is cancelled.
loss_params (dict): Parameters for helper.get_loss_base function to build loss.
- loss_params['pred_targets'] (a string or a list of strings):
contain the names of inputs nodes that will be sent into the loss function
- loss_params['loss_func']:
the function used to calculate the loss. Must be provided.
- loss_params['loss_func_kwargs'] (dict):
Keyword parameters sent to ``loss_params['loss_func']``. Default is {}.
- loss_params['agg_func']:
The aggregate function, default is None.
- loss_params['agg_func_kwargs']:
Keyword parameters sent to ``loss_params['agg_func']``. Default is {}.
- loss_params['loss_per_case_func'] (Deprecated):
Deprecated parameter, the same as ``loss_params['loss_func']``.
- loss_params['targets'] (Deprecated):
Deprecated parameter, the same as ``loss_params['targets']``.
learning_rate_params (dict): Parameters for specifying learning_rate.
- learning_rate_params['func']:
The function producing tensorflow node acting as learning rate.
This function must accept argument ``global_step``.
- remainder of learning_rate_params are arguments to func.
optimizer_params (dict): Parameters for creating optimizer.
- optimizer_params['optimizer']:
A class producing an optimizer object,
which should have function ``compute_gradients`` and ``apply_gradients``.
The signatures of these two functions are similar as tensorflow basic optimizer classes.
Must accept:
- "learning_rate" -- the result of the learning_rate_func call
- Remainder of optimizer_params (aside form "optimizer") are arguments
to the optimizer func
- optimizer_params['func'] (Deprecated):
Deprecated parameter, the same as ``optimizer_params['optimizer']``.
validation_params (dict): Dictionary of validation sources. The structure if this dictionary is:
{
<validation_target_name_1>: {
data: {
'func': (callable) data source function for this validation,
<kwarg1>: <value1> for 'func',
...
},
targets: {
'func': (callable) returning targets,
<kwarg1>: <value1> for 'func',
...
},
num_steps (int):
number of batches of validation source to compute,
agg_func (optional, callable):
how to aggregate validation results
across batches after computation. Signature is:
- one input argument: the list of validation batch results
- one output: aggregated version
Default is ``utils.identity_func``
online_agg_func (optional, callable):
how to aggregate validation results
on a per-batch basis. Siganture is:
- three input arguments: (current aggregate, new result, step)
- one output: new aggregated result
On first step, current aggregate passed in is None.
The final result is passed to the "agg_func".
Default is ``utils.append_and_return``
},
<validation_target_name_2>: ...
}
For each validation_target_name key, the targets are computed and then added to
the output dictionary to be computed every so often -- unlike train_targets which
are computed on each time step, these are computed on a basic controlled by the
valid_save_freq specific in the save_params.
load_params (dict):
Similar to save_params, if you want loading to happen from a different
location than where saving occurs. Parameters include:
- host (str)
Hostname where database connection lives
- port (int)
Port where database connection lives
- dbname (str)
Name of database for storage
- collname (str)
Name of collection for storage
- exp_id (str)
Experiment id descriptor
- do_restore (bool, default: True)
Whether to restore from saved model
- query (dict)
mongodb query describing how to load from loading database
- from_ckpt (string)
Path to load from a TensorFlow checkpoint (instead of from the db)
- to_restore (list of strings or a regex/callable which returns strings)
Specifies which variables should be loaded from the checkpoint.
Any variables not specified here will be reinitialized.
- load_param_dict (dict)
A dictionary whose keys are the names of the variables that are to be loaded
from the checkpoint, and the values are the names of the variables of the model
that you want to restore with the value of the corresponding checkpoint variable.
log_device_placement (bool, default is False):
Advanced parameter. Whether to log device placement in tensorflow session
dont_run (bool, default is False):
Advanced parameter. Whether returning everything, not actually training
skip_check (bool, default is False):
Advanced parameter. Whether skipping github check, could be useful when working in detached head
"""
params, train_args = parse_params(
'train',
model_params,
dont_run=dont_run,
skip_check=skip_check,
load_params=load_params,
loss_params=loss_params,
save_params=save_params,
train_params=train_params,
optimizer_params=optimizer_params,
validation_params=validation_params,
learning_rate_params=learning_rate_params,
log_device_placement=log_device_placement,
)
with tf.Graph().as_default(), tf.device(DEFAULT_HOST):
# For convenience, use list of dicts instead of dict of lists
_params = [{key: value[i]
for (key, value) in params.items()}
for i in range(len(params['model_params']))]
_trargs = [{key: value[i]
for (key, value) in train_args.items()}
for i in range(len(params['model_params']))]
# Use a single dataprovider for all models.
data_params = _params[0]['train_params']['data_params']
_params[0]['train_params']['data_params'], inputs = get_data(
**data_params)
# Build a graph for each distinct model.
for param, trarg in zip(_params, _trargs):
with tf.variable_scope(param['model_params']['prefix']):
# Build global step
trarg['global_step'] = tf.get_variable(
'global_step', [],
dtype=tf.int64,
trainable=False,
initializer=tf.constant_initializer(0))
_, _, param, trarg = get_model(inputs,
param['model_params'],
param=param,
trarg=trarg)
tf.get_variable_scope().reuse_variables()
trarg['validation_targets'] = \
get_valid_targets_dict(
**param)
# Create session.
gpu_options = tf.GPUOptions(allow_growth=True)
sess = tf.Session(config=tf.ConfigProto(
allow_soft_placement=True,
gpu_options=gpu_options,
log_device_placement=log_device_placement,
))
init_op_global = tf.global_variables_initializer()
sess.run(init_op_global)
init_op_local = tf.local_variables_initializer()
sess.run(init_op_local)
log.info('Initialized from scratch first')
for param, trarg in zip(_params, _trargs):
prefix = param['model_params']['prefix'] + '/'
all_vars = variables._all_saveable_objects()
var_list = strip_prefix(prefix, all_vars)
for var in var_list:
print(var)
trarg['dbinterface'] = DBInterface(
sess=sess,
params=param,
var_list=var_list,
global_step=trarg['global_step'],
save_params=param['save_params'],
load_params=param['load_params'])
trarg['dbinterface'].initialize()
# Convert back to a dictionary of lists
params = {
key: [param[key] for param in _params]
for key in _params[0].keys()
}
train_args = {
key: [trarg[key] for trarg in _trargs]
for key in _trargs[0].keys()
}
if dont_run:
return train_args
return train(sess, **train_args)
def add_meta_graph(self,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None,
strip_default_attrs=False):
# pylint: disable=line-too-long
"""Adds the current meta graph to the SavedModel.
Creates a Saver in the current scope and uses the Saver to export the meta
graph def. Invoking this API requires the `add_meta_graph_and_variables()`
API to have been invoked before.
Args:
tags: The set of tags to annotate the meta graph def with.
signature_def_map: The map of signature defs to be added to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel. Note
that this collection should be a subset of the assets saved as part of
the first meta graph in the SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded. Note
that when the main_op is specified it is run after the restore op at
load-time.
strip_default_attrs: Boolean. If `True`, default-valued attributes will be
removed from the NodeDefs. For a detailed guide, see
[Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes).
Raises:
AssertionError: If the variables for the SavedModel have not been saved
yet, or if the graph already contains one or more legacy init ops.
"""
# pylint: enable=line-too-long
if not self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has not been saved yet. "
"Please invoke `add_meta_graph_and_variables()` first.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Add assets and ops
self._add_collections(assets_collection, legacy_init_op, main_op, None)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. Removing the preexisting ones was the
# motivation for the clear_extraneous_savers option, but it turns out that
# there are edge cases where that option breaks the graph. Until that is
# resolved, we just leave the option set to False for now.
# TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
meta_graph_def = saver.export_meta_graph(
clear_devices=clear_devices, strip_default_attrs=strip_default_attrs)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
groundtruth_negatives = tf.placeholder(shape=[None, all_default_boxs_len], dtype=tf.float32,
name='groundtruth_negatives')
groundtruth_count = tf.add(groundtruth_positives, groundtruth_negatives)
learning_rt=0.000001
learning_rate = tf.placeholder(tf.float32, None, 'learning_rate')
loss_allt, loss_classt, loss_locationt = elloss(fc, fl, groundtruth_class, groundtruth_location, groundtruth_positives, groundtruth_count)
train = tf.train.MomentumOptimizer(learning_rate,momentum=0.9).minimize(loss_allt)
tf.summary.scalar('loss_all_train', loss_allt)
tf.summary.scalar('loss_class_train', tf.reduce_sum(loss_classt) )
tf.summary.scalar('loss_location_train', tf.reduce_sum(loss_locationt))
merged = tf.summary.merge_all()
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
trainwrite = tf.summary.FileWriter("logs/", sess.graph)
sess.run(tf.global_variables_initializer())
saver2 = tf.train.Saver(var_list=tf.trainable_variables())
zzz = variables._all_saveable_objects().copy()
print(zzz)
saver = tf.train.Saver()
if os.path.exists('./session_paramsdddaleasy/session2.ckpt.index') :
print('\nStart Restore')
saver2.restore(sess, './session_paramsdddaleasy/session2.ckpt')
print('\nEnd Restore')
print('\nStart Training')
min_loss_location = 100000.
min_loss_class = 100000.
avg_loss=0
avg_lossloc=0
avg_losclass=0
ptlos=0
ptlosc=0
ptlosl=0
def add_meta_graph(self,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None,
strip_default_attrs=False):
# pylint: disable=line-too-long
"""Adds the current meta graph to the SavedModel.
Creates a Saver in the current scope and uses the Saver to export the meta
graph def. Invoking this API requires the `add_meta_graph_and_variables()`
API to have been invoked before.
Args:
tags: The set of tags to annotate the meta graph def with.
signature_def_map: The map of signature defs to be added to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel. Note
that this collection should be a subset of the assets saved as part of
the first meta graph in the SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded. Note
that when the main_op is specified it is run after the restore op at
load-time.
strip_default_attrs: Boolean. If `True`, default-valued attributes will be
removed from the NodeDefs. For a detailed guide, see
[Stripping Default-Valued Attributes](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md#stripping-default-valued-attributes).
Raises:
AssertionError: If the variables for the SavedModel have not been saved
yet, or if the graph already contains one or more legacy init ops.
"""
# pylint: enable=line-too-long
if not self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has not been saved yet. "
"Please invoke `add_meta_graph_and_variables()` first.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Add assets and ops
self._add_collections(assets_collection, legacy_init_op, main_op, None)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. Removing the preexisting ones was the
# motivation for the clear_extraneous_savers option, but it turns out that
# there are edge cases where that option breaks the graph. Until that is
# resolved, we just leave the option set to False for now.
# TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
meta_graph_def = saver.export_meta_graph(
clear_devices=clear_devices,
strip_default_attrs=strip_default_attrs)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
img_train_batch, label_train_batch = tf.train.shuffle_batch(
[img_train, label_train], batch_size=batch_size, capacity=BATCH_CAPACITY,
min_after_dequeue=MIN_AFTER_DEQU)
img_test_batch, label_test_batch = tf.train.shuffle_batch(
[img_test,label_test], batch_size=batch_size, capacity=BATCH_CAPACITY,
min_after_dequeue=MIN_AFTER_DEQU)
net,logits_train=inference(img_train_batch,True,None)
_,logits_test=inference(img_test_batch,False,True)
loss_train=cal_loss(logits_train,label_train_batch)
loss_test=cal_loss(logits_test,label_test_batch)
acc_test=cal_acc(logits_test,label_test_batch)
acc_train=cal_acc(logits_train,label_train_batch)
train_mean,train_var=cal_mean_var(logits_train,label_train_batch)
print(train_mean,train_var)
test_mean,test_var=cal_mean_var_alter(logits_test,label_test_batch)
all_var = variables._all_saveable_objects().copy()
lens_all=len(all_var)
for _ in range(lens_all-2):
del all_var[0]
print(all_var)
pre_train = tf.train.MomentumOptimizer(0.005,momentum=0.9).minimize(loss_train,var_list=all_var)
global_step=tf.train.create_global_step()
global_step=tf.train.get_global_step()
learning_rate=tf.train.exponential_decay(learning_rt,global_step,
10000, 0.9, staircase=True)
train = tf.train.MomentumOptimizer(learning_rate,momentum=0.9).minimize(loss_train,global_step=global_step)
# train = tf.train.AdamOptimizer(learning_rt).minimize(loss_train)
tf.summary.scalar('loss_train', loss_train)
tf.summary.scalar('acc_train', acc_train)
merged = tf.summary.merge_all()
def add_meta_graph(self,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None):
"""Adds the current meta graph to the SavedModel.
Creates a Saver in the current scope and uses the Saver to export the meta
graph def. Invoking this API requires the `add_meta_graph_and_variables()`
API to have been invoked before.
Args:
tags: The set of tags to annotate the meta graph def with.
signature_def_map: The map of signature defs to be added to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel. Note
that this collection should be a subset of the assets saved as part of
the first meta graph in the SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded. Note
that when the main_op is specified it is run after the restore op at
load-time.
Raises:
AssertionError: If the variables for the SavedModel have not been saved
yet.
"""
if not self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has not been saved yet. "
"Please invoke `add_meta_graph_and_variables()` first.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Save asset files and write them to disk, if any.
self._save_and_write_assets(assets_collection)
if main_op is None:
# Add legacy init op to the SavedModel.
self._maybe_add_legacy_init_op(legacy_init_op)
else:
self._add_main_op(main_op)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. Removing the preexisting ones was the
# motivation for the clear_extraneous_savers option, but it turns out that
# there are edge cases where that option breaks the graph. Until that is
# resolved, we just leave the option set to False for now.
# TODO(soergel): Reinstate clear_extraneous_savers=True when possible.
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
groundtruth_negatives = tf.placeholder(shape=[None, all_default_boxs_len], dtype=tf.float32,
name='groundtruth_negatives')
groundtruth_count = tf.add(groundtruth_positives, groundtruth_negatives)
learning_rt=0.000001
learning_rate = tf.placeholder(tf.float32, None, 'learning_rate')
loss_allt, loss_classt, loss_locationt = elloss(fc, fl, groundtruth_class, groundtruth_location, groundtruth_positives, groundtruth_count)
train = tf.train.MomentumOptimizer(learning_rate,momentum=0.9).minimize(loss_allt)
tf.summary.scalar('loss_all_train', loss_allt)
tf.summary.scalar('loss_class_train', tf.reduce_sum(loss_classt) )
tf.summary.scalar('loss_location_train', tf.reduce_sum(loss_locationt))
merged = tf.summary.merge_all()
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
trainwrite = tf.summary.FileWriter("logs/", sess.graph)
sess.run(tf.global_variables_initializer())
saver2 = tf.train.Saver(var_list=tf.trainable_variables())
zzz = variables._all_saveable_objects().copy()
print(zzz)
saver = tf.train.Saver()
if os.path.exists('./session_paramsdddaleasy/session2.ckpt.index') :
print('\nStart Restore')
saver2.restore(sess, './session_paramsdddaleasy/session2.ckpt')
print('\nEnd Restore')
print('\nStart Training')
min_loss_location = 100000.
min_loss_class = 100000.
avg_loss=0
avg_lossloc=0
avg_losclass=0
ptlos=0
ptlosc=0
ptlosl=0
test = read_conf('dict2.txt')
n_step_epoch = int(int((train[0].split(':'))[1]) / batch_size)
n_step = n_epoch * n_step_epoch
with tf.device('/gpu:0'): # 使用GPU
# 在GPU上训练
train_params = network.all_params # 待训练的参数为所有的网络参数
# 定义训练操作，使用自适应矩估计（ADAM）算法最小化损失函数
train_op = tf.train.AdamOptimizer(learning_rate,
beta1=0.9,
beta2=0.999,
epsilon=1e-08,
use_locking=False).minimize(cost)
tl.layers.initialize_global_variables(sess)
print(variables._all_saveable_objects())
if resume:
print("Load existing model " + "!" * 10)
saver = tf.train.Saver()
saver.restore(sess, model_file_name)
network.print_params(False)
network.print_layers()
print(' learning_rate: %f' % learning_rate)
print(' batch_size: %d' % batch_size)
print(' n_epoch: %d, step in an epoch: %d, total n_step: %d' %
(n_epoch, n_step_epoch, n_step))
coord = tf.train.Coordinator() # 创建一个线程协调器
threads = tf.train.start_queue_runners(sess=sess, coord=coord) # 创建线程
def add_meta_graph(self,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None):
"""Adds the current meta graph to the SavedModel.
Creates a Saver in the current scope and uses the Saver to export the meta
graph def. Invoking this API requires the `add_meta_graph_and_variables()`
API to have been invoked before.
Args:
tags: The set of tags to annotate the meta graph def with.
signature_def_map: The map of signature defs to be added to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel. Note
that this collection should be a subset of the assets saved as part of
the first meta graph in the SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded.
Raises:
AssertionError: If the variables for the SavedModel have not been saved
yet.
"""
if not self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has not been saved yet. "
"Please invoke `add_meta_graph_and_variables()` first.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Save asset files and write them to disk, if any.
self._save_and_write_assets(assets_collection)
if main_op is None:
# Add legacy init op to the SavedModel.
self._maybe_add_legacy_init_op(legacy_init_op)
else:
self._add_main_op(main_op)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. In the context of the SavedModel, this
# new Saver is the only one that needs to be retained. The associated
# checkpoint produced in add_meta_graph_and_variables() contains all of the
# variable values. Thus, any preexisting Savers are redundant and useless
# at best, but worse may break downstream graph-processing tools, and can be
# confusing during debugging. It is therefore safe and wise to set
# `clear_extraneous_savers` to `True`, since it removes both the extraneous
# SaverDefs and their associated Save/Restore Ops from the graph.
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices,
clear_extraneous_savers=True)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
def add_meta_graph(self,
tags,
signature_def_map=None,
assets_collection=None,
legacy_init_op=None,
clear_devices=False,
main_op=None):
"""Adds the current meta graph to the SavedModel.
Creates a Saver in the current scope and uses the Saver to export the meta
graph def. Invoking this API requires the `add_meta_graph_and_variables()`
API to have been invoked before.
Args:
tags: The set of tags to annotate the meta graph def with.
signature_def_map: The map of signature defs to be added to the meta graph
def.
assets_collection: Assets collection to be saved with SavedModel. Note
that this collection should be a subset of the assets saved as part of
the first meta graph in the SavedModel.
legacy_init_op: Legacy support for op or group of ops to execute after the
restore op upon a load.
clear_devices: Set to true if the device info on the default graph should
be cleared.
main_op: Op or group of ops to execute when the graph is loaded.
Raises:
AssertionError: If the variables for the SavedModel have not been saved
yet.
"""
if not self._has_saved_variables:
raise AssertionError(
"Graph state including variables and assets has not been saved yet. "
"Please invoke `add_meta_graph_and_variables()` first.")
# Validate the signature def map to ensure all included TensorInfos are
# properly populated.
self._validate_signature_def_map(signature_def_map)
# Save asset files and write them to disk, if any.
self._save_and_write_assets(assets_collection)
if main_op is None:
# Add legacy init op to the SavedModel.
self._maybe_add_legacy_init_op(legacy_init_op)
else:
self._add_main_op(main_op)
# Initialize a saver to generate a sharded output for all saveables in the
# current scope.
saver = tf_saver.Saver(
variables._all_saveable_objects(), # pylint: disable=protected-access
sharded=True,
write_version=saver_pb2.SaverDef.V2,
allow_empty=True)
# The graph almost certainly previously contained at least one Saver, and
# possibly several (e.g. one for loading a pretrained embedding, and another
# for the model weights). However, a *new* Saver was just created that
# includes all of the variables. In the context of the SavedModel, this
# new Saver is the only one that needs to be retained. The associated
# checkpoint produced in add_meta_graph_and_variables() contains all of the
# variable values. Thus, any preexisting Savers are redundant and useless
# at best, but worse may break downstream graph-processing tools, and can be
# confusing during debugging. It is therefore safe and wise to set
# `clear_extraneous_savers` to `True`, since it removes both the extraneous
# SaverDefs and their associated Save/Restore Ops from the graph.
meta_graph_def = saver.export_meta_graph(clear_devices=clear_devices,
clear_extraneous_savers=True)
# Tag the meta graph def and add it to the SavedModel.
self._tag_and_add_meta_graph(meta_graph_def, tags, signature_def_map)
def test_from_params(load_params,
model_params,
validation_params,
log_device_placement=False,
save_params=None,
dont_run=False,
skip_check=False,
):
"""
Main testing interface function.
Same as train_from_parameters; but just performs testing without training.
For documentation, see argument descriptions in train_from_params.
"""
params, test_args = parse_params(
'test',
model_params,
dont_run=dont_run,
skip_check=skip_check,
save_params=save_params,
load_params=load_params,
validation_params=validation_params,
log_device_placement=log_device_placement,
)
with tf.Graph().as_default(), tf.device(DEFAULT_HOST):
# create session
sess = tf.Session(
config=tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=log_device_placement,
))
init_op_global = tf.global_variables_initializer()
sess.run(init_op_global)
init_op_local = tf.local_variables_initializer()
sess.run(init_op_local)
log.info('Initialized from scratch first')
# For convenience, use list of dicts instead of dict of lists
_params = [{key: value[i] for (key, value) in params.items()}
for i in range(len(params['model_params']))]
_ttargs = [{key: value[i] for (key, value) in test_args.items()}
for i in range(len(params['model_params']))]
# Build a graph for each distinct model.
for param, ttarg in zip(_params, _ttargs):
if not 'cache_dir' in load_params:
temp_cache_dir = save_params.get('cache_dir', None)
load_params['cache_dir'] = temp_cache_dir
log.info('cache_dir not found in load_params, using cache_dir ({}) from save_params'.format(temp_cache_dir))
ttarg['dbinterface'] = DBInterface(params=param, load_params=param['load_params'])
ttarg['dbinterface'].load_rec()
ld = ttarg['dbinterface'].load_data
assert ld is not None, "No load data found for query, aborting"
ld = ld[0]
# TODO: have option to reconstitute model_params entirely from
# saved object ("revivification")
param['model_params']['seed'] = ld['params']['model_params']['seed']
cfg_final = ld['params']['model_params']['cfg_final']
ttarg['validation_targets'] = \
get_valid_targets_dict(
loss_params=None,
cfg_final=cfg_final,
**param)
# tf.get_variable_scope().reuse_variables()
param['load_params']['do_restore'] = True
param['model_params']['cfg_final'] = cfg_final
prefix = param['model_params']['prefix'] + '/'
all_vars = variables._all_saveable_objects()
var_list = strip_prefix(prefix, all_vars)
ttarg['dbinterface'] = DBInterface(sess=sess,
params=param,
var_list=var_list,
load_params=param['load_params'],
save_params=param['save_params'])
ttarg['dbinterface'].initialize(no_scratch=True)
ttarg['save_intermediate_freq'] = param['save_params'].get('save_intermediate_freq')
# Convert back to a dictionary of lists
params = {key: [param[key] for param in _params]
for key in _params[0].keys()}
test_args = {key: [ttarg[key] for ttarg in _ttargs]
for key in _ttargs[0].keys()}
if dont_run:
return test_args
res = test(sess, **test_args)
sess.close()
return res
def _get_saveable_variables(exclude_scopes=tuple()):
# noinspection PyProtectedMember
all_vars = variables._all_saveable_objects()
vars_to_train = [var for var in all_vars if all(sc not in var.name for sc in exclude_scopes)]
return vars_to_train
def _get_saveable_variables(exclude_scopes=tuple()):
all_vars = variables._all_saveable_objects()
vars_to_train = [var for var in all_vars if all(sc not in var.name for sc in exclude_scopes)]
return vars_to_train
