def _get_signature_def(signature_def_key, export_dir, tags):
"""Construct a `SignatureDef` proto."""
signature_def_key = (signature_def_key or
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY)
metagraph_def = saved_model_cli.get_meta_graph_def(export_dir, tags)
try:
signature_def = signature_def_utils.get_signature_def_by_key(
metagraph_def, signature_def_key)
except ValueError as e:
try:
formatted_key = _DEFAULT_INPUT_ALTERNATIVE_FORMAT.format(
signature_def_key)
signature_def = signature_def_utils.get_signature_def_by_key(
metagraph_def, formatted_key)
logging.warning(
'Could not find signature def "%s". '
'Using "%s" instead', signature_def_key, formatted_key)
except ValueError:
raise ValueError(
'Got signature_def_key "{}". Available signatures are {}. '
'Original error:\n{}'.format(signature_def_key,
list(metagraph_def.signature_def),
e))
return signature_def
def load_meta_graph(model_path, tags, graph, signature_def_key=None):
saved_model = reader.read_saved_model(model_path)
the_meta_graph = None
for meta_graph_def in saved_model.meta_graphs:
if sorted(meta_graph_def.meta_info_def.tags) == sorted(tags):
the_meta_graph = meta_graph_def
signature_def_key = signature_def_key or tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
try:
signature_def = signature_def_utils.get_signature_def_by_key(
the_meta_graph,
signature_def_key)
except ValueError as ex:
try:
formatted_key = 'default_input_alternative:{}'.format(
signature_def_key)
signature_def = signature_def_utils.get_signature_def_by_key(
the_meta_graph, formatted_key)
except ValueError:
raise ValueError(
'Got signature_def_key "{}". Available signatures are {}. '
'Original error:\n{}'.format(
signature_def_key, list(the_meta_graph.signature_def), ex)
)
input_names = {k: v.name for k, v in signature_def.inputs.items()}
output_names = {k: v.name for k, v in signature_def.outputs.items()}
feed_tensors = {k: graph.get_tensor_by_name(v)
for k, v in input_names.items()}
fetch_tensors = {k: graph.get_tensor_by_name(v)
for k, v in output_names.items()}
return feed_tensors, fetch_tensors
def _get_signature_def(signature_def_key, export_dir, tags):
"""Construct a `SignatureDef` proto."""
signature_def_key = (
signature_def_key or
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY)
metagraph_def = saved_model_cli.get_meta_graph_def(export_dir, tags)
try:
signature_def = signature_def_utils.get_signature_def_by_key(
metagraph_def,
signature_def_key)
except ValueError as e:
try:
formatted_key = _DEFAULT_INPUT_ALTERNATIVE_FORMAT.format(
signature_def_key)
signature_def = signature_def_utils.get_signature_def_by_key(
metagraph_def, formatted_key)
logging.warning('Could not find signature def "%s". '
'Using "%s" instead', signature_def_key, formatted_key)
except ValueError:
raise ValueError(
'Got signature_def_key "{}". Available signatures are {}. '
'Original error:\n{}'.format(
signature_def_key, list(metagraph_def.signature_def), e))
return signature_def
def testGetSignatureDefByKey(self):
x = array_ops.placeholder(dtypes.float32, 1, name="x")
x_tensor_info = utils.build_tensor_info(x)
y = array_ops.placeholder(dtypes.float32, name="y")
y_tensor_info = utils.build_tensor_info(y)
foo_signature_def = signature_def_utils.build_signature_def(
{"foo-input": x_tensor_info}, {"foo-output": y_tensor_info},
"foo-method-name")
bar_signature_def = signature_def_utils.build_signature_def(
{"bar-input": x_tensor_info}, {"bar-output": y_tensor_info},
"bar-method-name")
meta_graph_def = meta_graph_pb2.MetaGraphDef()
self._add_to_signature_def_map(meta_graph_def, {
"foo": foo_signature_def,
"bar": bar_signature_def
})
# Look up a key that does not exist in the SignatureDefMap.
missing_key = "missing-key"
with self.assertRaisesRegexp(
ValueError,
"No SignatureDef with key '%s' found in MetaGraphDef" %
missing_key):
signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, missing_key)
# Look up the key, `foo` which exists in the SignatureDefMap.
foo_signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "foo")
self.assertTrue("foo-method-name", foo_signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(1, len(foo_signature_def.inputs))
self._check_tensor_info(foo_signature_def.inputs, "foo-input", "x:0")
# Check outputs in signature def.
self.assertEqual(1, len(foo_signature_def.outputs))
self._check_tensor_info(foo_signature_def.outputs, "foo-output", "y:0")
# Look up the key, `bar` which exists in the SignatureDefMap.
bar_signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "bar")
self.assertTrue("bar-method-name", bar_signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(1, len(bar_signature_def.inputs))
self._check_tensor_info(bar_signature_def.inputs, "bar-input", "x:0")
# Check outputs in signature def.
self.assertEqual(1, len(bar_signature_def.outputs))
self._check_tensor_info(bar_signature_def.outputs, "bar-output", "y:0")
def testGetSignatureDefByKey(self):
x = array_ops.placeholder(dtypes.float32, 1, name="x")
x_tensor_info = utils.build_tensor_info(x)
y = array_ops.placeholder(dtypes.float32, name="y")
y_tensor_info = utils.build_tensor_info(y)
foo_signature_def = signature_def_utils.build_signature_def({
"foo-input": x_tensor_info
}, {"foo-output": y_tensor_info}, "foo-method-name")
bar_signature_def = signature_def_utils.build_signature_def({
"bar-input": x_tensor_info
}, {"bar-output": y_tensor_info}, "bar-method-name")
meta_graph_def = meta_graph_pb2.MetaGraphDef()
self._add_to_signature_def_map(
meta_graph_def, {"foo": foo_signature_def,
"bar": bar_signature_def})
# Look up a key that does not exist in the SignatureDefMap.
missing_key = "missing-key"
with self.assertRaisesRegexp(
ValueError,
"No SignatureDef with key '%s' found in MetaGraphDef" % missing_key):
signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, missing_key)
# Look up the key, `foo` which exists in the SignatureDefMap.
foo_signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "foo")
self.assertTrue("foo-method-name", foo_signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(1, len(foo_signature_def.inputs))
self._check_tensor_info(foo_signature_def.inputs, "foo-input", "x:0")
# Check outputs in signature def.
self.assertEqual(1, len(foo_signature_def.outputs))
self._check_tensor_info(foo_signature_def.outputs, "foo-output", "y:0")
# Look up the key, `bar` which exists in the SignatureDefMap.
bar_signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "bar")
self.assertTrue("bar-method-name", bar_signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(1, len(bar_signature_def.inputs))
self._check_tensor_info(bar_signature_def.inputs, "bar-input", "x:0")
# Check outputs in signature def.
self.assertEqual(1, len(bar_signature_def.outputs))
self._check_tensor_info(bar_signature_def.outputs, "bar-output", "y:0")
def test(test_input_filename, test_output_filename):
model_path = '.'
with open(test_input_filename, 'rb') as fh:
actual_input = fh.read()
with open(test_output_filename, 'rb') as fh:
expected_output = fh.read()
# Load io_transformers module
transformers_module_name = 'model_io_transformers'
spec = importlib.util.spec_from_file_location(transformers_module_name, '%s.py' % transformers_module_name)
transformers_module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(transformers_module)
actual_transformed_input = transformers_module.transform_inputs(actual_input)
print(actual_transformed_input)
# model.setup()
with tf.Session(graph=tf.Graph()) as sess:
meta_graph_def = tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], model_path)
print('meta_graph_def %s' % meta_graph_def)
print('graph %s' % sess.graph)
#x_observed = tf.placeholder(dtype=tf.float32,
# shape=[None, 1],
# name='x_observed')
internal_input_tensor_name = signature_def_contrib_utils.get_signature_def_by_key(meta_graph_def, 'predict').inputs['x_observed'].name
#x_observed = sess.graph.get_tensor_by_name('x_observed:0')
x_observed_internal = sess.graph.get_tensor_by_name(internal_input_tensor_name)
internal_output_tensor_name = signature_def_contrib_utils.get_signature_def_by_key(meta_graph_def, 'predict').outputs['y_pred'].name
#y_pred = sess.graph.get_tensor_by_name('add:0')
y_pred_internal = sess.graph.get_tensor_by_name(internal_output_tensor_name)
#actual_output = sess.run(y_pred, feed_dict={'x_observed:0':actual_transformed_input.inputs['x_observed'].float_val})
actual_output = sess.run(y_pred_internal, feed_dict={internal_input_tensor_name:actual_transformed_input.inputs['x_observed'].float_val})
print('actual output: %s' % actual_output)
print('actual output type: %s' % type(actual_output))
#actual_output = model.predict(actual_transformed_input)
#hacked_actual_output = type('hacked_actual_output_obj', (object,), {'outputs' : actual_output})
#hacked_actual_output.outputs = {'y_pred': actual_output}
actual_transformed_output = transformers_module.transform_outputs(actual_output)
#json.dumps(actual_output.tolist())
#transformers_module.transform_outputs(actual_output)
# TODO: add {"y_pred":...}
print('actual transformed output: %s' % actual_transformed_output)
print('expected: %s' % expected_output.decode('utf-8').strip())
return (json.loads(expected_output.decode('utf-8').strip()) == json.loads(actual_transformed_output.strip()))
def _from_saved_model_v1(sess, model_path, input_names, output_names, tag, signatures):
"""Load tensorflow graph from saved_model."""
if tag is None:
tag = [tf.saved_model.tag_constants.SERVING]
if not isinstance(tag, list):
tag = [tag]
imported = tf.saved_model.loader.load(sess, tag, model_path)
for k in imported.signature_def.keys():
if k.startswith("_"):
# consider signatures starting with '_' private
continue
signatures.append(k)
try:
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
# pylint: disable=unnecessary-lambda
get_signature_def = lambda meta_graph_def, k: \
signature_def_utils.get_signature_def_by_key(meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[k]
input_names = []
output_names = []
for k in signatures:
inputs_tensor_info = get_signature_def(imported, k).inputs
for _, input_tensor in inputs_tensor_info.items():
input_names.append(input_tensor.name)
outputs_tensor_info = get_signature_def(imported, k).outputs
for _, output_tensor in outputs_tensor_info.items():
output_names.append(output_tensor.name)
frozen_graph = freeze_session(sess, input_names=input_names, output_names=output_names)
return frozen_graph, input_names, output_names
def testGetSignatureDefByKeyRegression(self):
input1 = constant_op.constant("a", name="input-1")
output1 = constant_op.constant(7.2, name="output-1")
meta_graph_def = meta_graph_pb2.MetaGraphDef()
self._add_to_signature_def_map(meta_graph_def, {
"my_regression":
signature_def_utils.regression_signature_def(input1, output1)
})
# Look up the regression signature with the key used while saving.
signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "my_regression")
# Check the method name to match the constants regression method name.
self.assertEqual(signature_constants.REGRESS_METHOD_NAME,
signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(1, len(signature_def.inputs))
self._check_tensor_info(signature_def.inputs,
signature_constants.REGRESS_INPUTS, "input-1:0")
# Check outputs in signature def.
self.assertEqual(1, len(signature_def.outputs))
self._check_tensor_info(signature_def.outputs,
signature_constants.REGRESS_OUTPUTS, "output-1:0")
def _get_signature_def(meta_graph, signature_key):
"""Get the signature def from meta_graph with given signature_key.
Args:
meta_graph: meta_graph_def.
signature_key: signature_def in the meta_graph_def.
Returns:
The signature_def used for tflite conversion.
Raises:
ValueError: Given signature_key is not valid for this meta_graph.
"""
signature_def_map = meta_graph.signature_def
signature_def_keys = set(signature_def_map.keys())
logging.info(
"The given SavedModel MetaGraphDef contains SignatureDefs with the "
"following keys: %s", signature_def_keys)
if signature_key not in signature_def_keys:
raise ValueError("No '{}' in the SavedModel\'s SignatureDefs. Possible "
"values are '{}'.".format(signature_key,
",".join(signature_def_keys)))
signature_def = signature_def_utils.get_signature_def_by_key(
meta_graph, signature_key)
return signature_def
def RunModel(saved_model_dir, signature_def_key, tag, text, ngrams_list=None):
saved_model = reader.read_saved_model(saved_model_dir)
meta_graph = None
for meta_graph_def in saved_model.meta_graphs:
if tag in meta_graph_def.meta_info_def.tags:
meta_graph = meta_graph_def
break
if meta_graph_def is None:
raise ValueError("Cannot find saved_model with tag" + tag)
signature_def = signature_def_utils.get_signature_def_by_key(
meta_graph, signature_def_key)
text = text_utils.TokenizeText(text)
ngrams = None
if ngrams_list is not None:
ngrams_list = text_utils.ParseNgramsOpts(ngrams_list)
ngrams = text_utils.GenerateNgrams(text, ngrams_list)
example = inputs.BuildTextExample(text, ngrams=ngrams)
example = example.SerializeToString()
inputs_feed_dict = {
signature_def.inputs["inputs"].name: [example],
}
if signature_def_key == "proba":
output_key = "scores"
elif signature_def_key == "embedding":
output_key = "outputs"
else:
raise ValueError("Unrecognised signature_def %s" % (signature_def_key))
output_tensor = signature_def.outputs[output_key].name
with tf.Session() as sess:
loader.load(sess, [tag], saved_model_dir)
outputs = sess.run(output_tensor, feed_dict=inputs_feed_dict)
return outputs
def __init__(self, path=DEFAULT_MODEL_PATH, model_dir=DEFAULT_MODEL_DIR):
logger.info('Loading model from: {}...'.format(path))
sess = tf.Session(graph=tf.Graph())
# load the graph
saved_model_path = '{}/{}'.format(path, model_dir)
model_graph_def = sm.loader.load(sess, [sm.tag_constants.SERVING],
saved_model_path)
sig_def = signature_def_utils.get_signature_def_by_key(
model_graph_def,
sm.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY)
input_name = sig_def.inputs['inputs'].name
output_name = sig_def.outputs['scores'].name
# Load labels from file
label_file = codecs.open('./{}/labels.txt'.format(path),
"r",
encoding="utf-8")
labels = [label.strip('\n') for label in label_file.readlines()]
self.labels = labels
# set up instance variables and required inputs for inference
self.sess = sess
self.model_graph_def = model_graph_def
self.output_tensor = sess.graph.get_tensor_by_name(output_name)
self.input_name = input_name
self.output_name = output_name
self.means = np.load('./{}/crop_mean.npy'.format(path)).reshape(
[NUM_FRAMES_PER_CLIP, CROP_SIZE, CROP_SIZE, CHANNELS])
logger.info('Loaded model')
def _get_signature_def(meta_graph, signature_key):
"""Get the signature def from meta_graph with given signature_key.
Args:
meta_graph: meta_graph_def.
signature_key: signature_def in the meta_graph_def.
Returns:
The signature_def used for tflite conversion.
Raises:
ValueError: Given signature_key is not valid for this meta_graph.
"""
signature_def_map = meta_graph.signature_def
signature_def_keys = set(signature_def_map.keys())
logging.info(
"The given saved_model MetaGraphDef contains SignatureDefs with the "
"following keys: %s", signature_def_keys)
if signature_key not in signature_def_keys:
raise ValueError(
"No '{}' in the saved_model\'s SignatureDefs. Possible "
"values are '{}'. ".format(signature_key, signature_def_keys))
signature_def = signature_def_utils.get_signature_def_by_key(
meta_graph, signature_key)
return signature_def
def testGetSignatureDefByKeyRegression(self):
input1 = constant_op.constant("a", name="input-1")
output1 = constant_op.constant("b", name="output-1")
meta_graph_def = meta_graph_pb2.MetaGraphDef()
self._add_to_signature_def_map(
meta_graph_def, {
"my_regression":
signature_def_utils.regression_signature_def(input1, output1)
})
# Look up the regression signature with the key used while saving.
signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "my_regression")
# Check the method name to match the constants regression method name.
self.assertEqual(signature_constants.REGRESS_METHOD_NAME,
signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(1, len(signature_def.inputs))
self._check_tensor_info(signature_def.inputs,
signature_constants.REGRESS_INPUTS,
"input-1:0")
# Check outputs in signature def.
self.assertEqual(1, len(signature_def.outputs))
self._check_tensor_info(signature_def.outputs,
signature_constants.REGRESS_OUTPUTS,
"output-1:0")
def from_saved_model(model_path, input_names, output_names):
"""Load tensorflow graph from saved_model."""
# make sure we start with clean default graph
tf.reset_default_graph()
inputs = {}
outputs = {}
try:
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
# pylint: disable=unnecessary-lambda
get_signature_def = lambda meta_graph_def, k: \
signature_def_utils.get_signature_def_by_key(meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[
k]
with tf.Session() as sess:
meta_graph_def = tf.saved_model.loader.load(
sess, [tf.saved_model.tag_constants.SERVING], model_path)
for k in meta_graph_def.signature_def.keys():
inputs_tensor_info = get_signature_def(meta_graph_def, k).inputs
for _, input_tensor in sorted(inputs_tensor_info.items()):
inputs[input_tensor.name] = sess.graph.get_tensor_by_name(
input_tensor.name)
outputs_tensor_info = get_signature_def(meta_graph_def, k).outputs
for _, output_tensor in sorted(outputs_tensor_info.items()):
outputs[output_tensor.name] = sess.graph.get_tensor_by_name(
output_tensor.name)
frozen_graph = freeze_session(sess, output_names=list(outputs.keys()))
if input_names is None:
input_names = inputs.keys()
input_names = remove_redundant_inputs(frozen_graph, input_names)
# clean up
tf.reset_default_graph()
return frozen_graph, input_names, outputs.keys()
def testGetSignatureDefByKeyClassification(self):
input1 = constant_op.constant("a", name="input-1")
output1 = constant_op.constant("b", name="output-1")
output2 = constant_op.constant("c", name="output-2")
meta_graph_def = meta_graph_pb2.MetaGraphDef()
self._add_to_signature_def_map(
meta_graph_def, {
"my_classification":
signature_def_utils.classification_signature_def(
input1, output1, output2)
})
# Look up the classification signature def with the key used while saving.
signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "my_classification")
# Check the method name to match the constants classification method name.
self.assertEqual(signature_constants.CLASSIFY_METHOD_NAME,
signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(1, len(signature_def.inputs))
self._check_tensor_info(signature_def.inputs,
signature_constants.CLASSIFY_INPUTS,
"input-1:0")
# Check outputs in signature def.
self.assertEqual(2, len(signature_def.outputs))
self._check_tensor_info(signature_def.outputs,
signature_constants.CLASSIFY_OUTPUT_CLASSES,
"output-1:0")
self._check_tensor_info(signature_def.outputs,
signature_constants.CLASSIFY_OUTPUT_SCORES,
"output-2:0")
def _from_saved_model_v1(sess, model_path, input_names, output_names, signatures):
"""Load tensorflow graph from saved_model."""
# make sure we start with clean default graph
inputs = {}
outputs = {}
imported = tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], model_path)
for k in imported.signature_def.keys():
if k.startswith("_"):
# consider signatures starting with '_' private
continue
signatures.append(k)
try:
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
# pylint: disable=unnecessary-lambda
get_signature_def = lambda meta_graph_def, k: \
signature_def_utils.get_signature_def_by_key(meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[k]
for k in signatures:
inputs_tensor_info = get_signature_def(imported, k).inputs
for _, input_tensor in inputs_tensor_info.items():
inputs[input_tensor.name] = sess.graph.get_tensor_by_name(input_tensor.name)
outputs_tensor_info = get_signature_def(imported, k).outputs
for _, output_tensor in outputs_tensor_info.items():
outputs[output_tensor.name] = sess.graph.get_tensor_by_name(output_tensor.name)
frozen_graph = freeze_session(sess, input_names=list(inputs.keys()), output_names=list(outputs.keys()))
return frozen_graph, inputs, outputs
def testGetSignatureDefByKeyClassification(self):
input1 = constant_op.constant("a", name="input-1")
output1 = constant_op.constant("b", name="output-1")
output2 = constant_op.constant(3.0, name="output-2")
meta_graph_def = meta_graph_pb2.MetaGraphDef()
self._add_to_signature_def_map(meta_graph_def, {
"my_classification":
signature_def_utils.classification_signature_def(
input1, output1, output2)
})
# Look up the classification signature def with the key used while saving.
signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "my_classification")
# Check the method name to match the constants classification method name.
self.assertEqual(signature_constants.CLASSIFY_METHOD_NAME,
signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(1, len(signature_def.inputs))
self._check_tensor_info(signature_def.inputs,
signature_constants.CLASSIFY_INPUTS, "input-1:0")
# Check outputs in signature def.
self.assertEqual(2, len(signature_def.outputs))
self._check_tensor_info(signature_def.outputs,
signature_constants.CLASSIFY_OUTPUT_CLASSES,
"output-1:0")
self._check_tensor_info(signature_def.outputs,
signature_constants.CLASSIFY_OUTPUT_SCORES,
"output-2:0")
def testPredictionSignatureDef(self):
input1 = constant_op.constant("a", name="input-1")
input2 = constant_op.constant("b", name="input-2")
output1 = constant_op.constant("c", name="output-1")
output2 = constant_op.constant("d", name="output-2")
meta_graph_def = meta_graph_pb2.MetaGraphDef()
self._add_to_signature_def_map(meta_graph_def, {
"my_prediction":
signature_def_utils.predict_signature_def({
"input-1": input1,
"input-2": input2
}, {"output-1": output1,
"output-2": output2})
})
# Look up the prediction signature def with the key used while saving.
signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "my_prediction")
self.assertEqual(signature_constants.PREDICT_METHOD_NAME,
signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(2, len(signature_def.inputs))
self._check_tensor_info(signature_def.inputs, "input-1", "input-1:0")
self._check_tensor_info(signature_def.inputs, "input-2", "input-2:0")
# Check outputs in signature def.
self.assertEqual(2, len(signature_def.outputs))
self._check_tensor_info(signature_def.outputs, "output-1", "output-1:0")
self._check_tensor_info(signature_def.outputs, "output-2", "output-2:0")
def _from_saved_model_v1(sess, model_path, input_names, output_names, tag, signature_names, use_graph_names):
"""Load tensorflow graph from saved_model."""
wrn_no_tag = "'--tag' not specified for saved_model. Using --tag serve"
wrn_empty_tag = "'--tag' value is empty string. Using tags = []"
wrn_empty_sig = "'--signature_def' not provided. Using all signatures."
if tag is None:
tag = [tf.saved_model.tag_constants.SERVING]
logger.warning(wrn_no_tag)
if not signature_names:
logger.warning(wrn_empty_sig)
if tag == '':
tag = []
logger.warning(wrn_empty_tag)
if not isinstance(tag, list):
tag = [tag]
imported = tf.saved_model.loader.load(sess, tag, model_path)
signatures = []
for k in imported.signature_def.keys():
if k in signature_names or (not signature_names and not k.startswith("_")):
signatures.append(k)
try:
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
# pylint: disable=unnecessary-lambda
get_signature_def = lambda meta_graph_def, k: \
signature_def_utils.get_signature_def_by_key(meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[k]
tensors_to_rename = {}
if input_names is None:
input_names = []
for k in signatures:
inputs_tensor_info = get_signature_def(imported, k).inputs
for structured_name, input_tensor in inputs_tensor_info.items():
if input_tensor.name not in input_names:
input_names.append(input_tensor.name)
if not use_graph_names:
tensors_to_rename[input_tensor.name] = structured_name
if output_names is None:
output_names = []
for k in signatures:
outputs_tensor_info = get_signature_def(imported, k).outputs
for structured_name, output_tensor in outputs_tensor_info.items():
if output_tensor.name not in output_names:
output_names.append(output_tensor.name)
if not use_graph_names:
tensors_to_rename[output_tensor.name] = structured_name
frozen_graph, initialized_tables = \
freeze_session(sess, input_names=input_names, output_names=output_names, get_tables=True)
return frozen_graph, input_names, output_names, initialized_tables, tensors_to_rename
def __init__(self):
model_path = os.environ.get('MODEL_PATH', '/model')
self.sess = tf.Session(graph=tf.Graph())
saved_metagraphdef = tf.saved_model.loader.load(
self.sess, [tag_constants.SERVING], model_path)
self.inputs_tensor_info = signature_def_utils.get_signature_def_by_key(
saved_metagraphdef,
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY).inputs
outputs_tensor_info = signature_def_utils.get_signature_def_by_key(
saved_metagraphdef,
signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY).outputs
self.output_tensor_keys_sorted = sorted(outputs_tensor_info.keys())
self.output_tensor_names_sorted = [
outputs_tensor_info[tensor_key].name
for tensor_key in self.output_tensor_keys_sorted
]
def _from_saved_model_v1(sess, model_path, tag, signatures):
"""
Load tensorflow graph from saved_model.
NOTICE: Modified version from tf2onnx project
"""
wrn_no_tag = "'--tag' not specified for saved_model. Using --tag serve"
wrn_empty_tag = "'--tag' value is empty string. Using tag =[[]]"
if tag is None:
tag = [tf.saved_model.SERVING]
LOGGER.warning(wrn_no_tag)
if tag == "":
tag = [[]]
LOGGER.warning(wrn_empty_tag)
if not isinstance(tag, list):
tag = [tag]
imported = tf.compat.v1.saved_model.loader.load(sess, tag, model_path)
for k in imported.signature_def.keys():
if k.startswith("_"):
# consider signatures starting with '_' private
continue
signatures.append(k)
try:
from tensorflow.contrib.saved_model.python.saved_model import ( # pytype: disable=import-error
signature_def_utils, )
# pylint: disable=unnecessary-lambda
get_signature_def = lambda meta_graph_def, k: signature_def_utils.get_signature_def_by_key(
meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[
k]
inputs = {}
outputs = {}
for k in signatures:
inputs_tensor_info = get_signature_def(imported, k).inputs
for name, input_tensor in inputs_tensor_info.items():
inputs[name] = input_tensor.name
outputs_tensor_info = get_signature_def(imported, k).outputs
for name, output_tensor in outputs_tensor_info.items():
outputs[name] = output_tensor.name
frozen_graph = freeze_session(sess,
input_names=list(inputs.values()),
output_names=list(outputs.values()))
return frozen_graph, inputs, outputs
def from_saved_model(model_path, input_names, output_names, signatures=None):
"""Load tensorflow graph from saved_model."""
# make sure we start with clean default graph
tf.reset_default_graph()
inputs = {}
outputs = {}
try:
# pylint: disable=C0415
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
# pylint: disable=unnecessary-lambda
get_signature_def = lambda meta_graph_def, k: \
signature_def_utils.get_signature_def_by_key(meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[
k]
with tf.Session() as sess:
meta_graph_def = tf.saved_model.loader.load(
sess, [tf.saved_model.tag_constants.SERVING], model_path)
if signatures is None:
signatures = []
for k in meta_graph_def.signature_def.keys():
if k.startswith("_"):
# consider signatures starting with '_' private
continue
signatures.append(k)
if len(signatures) > 1:
logger.warning(
"found multiple signatures %s in saved_model, pass --signature_def in command line",
signatures)
for k in signatures:
inputs_tensor_info = get_signature_def(meta_graph_def, k).inputs
for _, input_tensor in sorted(inputs_tensor_info.items()):
inputs[input_tensor.name] = sess.graph.get_tensor_by_name(
input_tensor.name)
outputs_tensor_info = get_signature_def(meta_graph_def, k).outputs
for _, output_tensor in sorted(outputs_tensor_info.items()):
outputs[output_tensor.name] = sess.graph.get_tensor_by_name(
output_tensor.name)
frozen_graph = freeze_session(sess, output_names=list(outputs.keys()))
if input_names is None:
input_names = inputs.keys()
input_names = remove_redundant_inputs(frozen_graph, input_names)
# clean up
tf.reset_default_graph()
return frozen_graph, input_names, list(outputs.keys())
def _get_outputs_tensor_info_from_meta_graph_def(meta_graph_def,
signature_def_key):
"""Gets TensorInfos for all outputs of the SignatureDef.
Returns a dictionary that maps each output key to its TensorInfo for the given
signature_def_key in the meta_graph_def.
Args:
meta_graph_def: MetaGraphDef protocol buffer with the SignatureDefmap to
look up signature_def_key.
signature_def_key: A SignatureDef key string.
Returns:
A dictionary that maps output tensor keys to TensorInfos.
"""
return signature_def_utils.get_signature_def_by_key(
meta_graph_def, signature_def_key).outputs
def _get_outputs_tensor_info_from_meta_graph_def(meta_graph_def,
signature_def_key):
"""Gets TensorInfos for all outputs of the SignatureDef.
Returns a dictionary that maps each output key to its TensorInfo for the given
signature_def_key in the meta_graph_def.
Args:
meta_graph_def: MetaGraphDef protocol buffer with the SignatureDefmap to
look up signature_def_key.
signature_def_key: A SignatureDef key string.
Returns:
A dictionary that maps output tensor keys to TensorInfos.
"""
return signature_def_utils.get_signature_def_by_key(meta_graph_def,
signature_def_key).outputs
def __init__(self, path=DEFAULT_MODEL_PATH):
logger.info('Loading model from: {}...'.format(path))
sess = tf.Session(graph=tf.Graph())
# Load the graph
model_graph_def = sm.loader.load(sess, [sm.tag_constants.SERVING], path)
sig_def = signature_def_utils.get_signature_def_by_key(model_graph_def,
sm.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY)
input_name = sig_def.inputs['input_images'].name
output_name = sig_def.outputs['output_images'].name
# Set up instance variables and required inputs for inference
self.sess = sess
self.model_graph_def = model_graph_def
self.output_tensor = sess.graph.get_tensor_by_name(output_name)
self.input_name = input_name
self.output_name = output_name
logger.info('Loaded model')
def __init__(self, model_path):
self.logger = logging.getLogger(self.__class__.__name__)
self.model_path = model_path
if not tf.saved_model.loader.maybe_saved_model_directory(self.model_path):
raise ValueError('No model found in', self.model_path)
self.sess = tf.Session(graph=tf.Graph())
meta_graph_def = tf.saved_model.loader.load(self.sess, [tf.saved_model.tag_constants.SERVING], self.model_path)
signature_def = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
meta_graph_def_sig = signature_def_utils.get_signature_def_by_key(meta_graph_def, signature_def)
self.input_tensor_info = meta_graph_def_sig.inputs
self.output_tensor_info = meta_graph_def_sig.outputs
self.output_tensor_keys = [k for k in self.output_tensor_info]
self.output_tensor_names = [self.output_tensor_info[k].name for k in self.output_tensor_keys]
def from_saved_model(model_path, input_names, output_names):
"""Load tensorflow graph from saved_model."""
# make sure we start with clean default graph
tf.reset_default_graph()
inputs = {}
outputs = {}
try:
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
# pylint: disable=unnecessary-lambda
get_signature_def = lambda meta_graph_def, k: \
signature_def_utils.get_signature_def_by_key(meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[
k]
with tf.Session() as sess:
meta_graph_def = tf.saved_model.loader.load(
sess, [tf.saved_model.tag_constants.SERVING], model_path)
for k in meta_graph_def.signature_def.keys():
inputs_tensor_info = get_signature_def(meta_graph_def, k).inputs
for _, input_tensor in sorted(inputs_tensor_info.items()):
inputs[input_tensor.name] = sess.graph.get_tensor_by_name(
input_tensor.name)
outputs_tensor_info = get_signature_def(meta_graph_def, k).outputs
for _, output_tensor in sorted(outputs_tensor_info.items()):
outputs[output_tensor.name] = sess.graph.get_tensor_by_name(
output_tensor.name)
frozen_graph = freeze_session(sess, output_names=list(outputs.keys()))
frozen_inputs = []
# get inputs in frozen graph
for n in frozen_graph.node:
for inp, _ in inputs.items():
if utils.node_name(inp) == n.name:
frozen_inputs.append(inp)
deleted_inputs = list(set(inputs.keys()) - set(frozen_inputs))
if deleted_inputs:
log.warning("inputs [%s] is not in frozen graph, delete them",
",".join(deleted_inputs))
# clean up
tf.reset_default_graph()
return frozen_graph, frozen_inputs, outputs.keys()
def RunModel(saved_model_dir, signature_def_key, feature_text, feature_map):
saved_model = reader.read_saved_model(saved_model_dir)
meta_graph = None
for meta_graph_def in saved_model.meta_graphs:
if meta_graph_def.meta_info_def.tags == _TAG:
meta_graph = meta_graph_def
signature_def = signature_def_utils.get_signature_def_by_key(
meta_graph, signature_def_key)
features = pi.get_feature_list(feature_map, feature_text.split(" "))
inputs_feed_dict = {signature_def.inputs["inputs"].name: features}
if signature_def_key == "proba":
output_key = "scores"
elif signature_def_key == "embedding":
output_key = "outputs"
else:
raise ValueError("Unrecognised signature_def %s" % (signature_def_key))
output_tensor = signature_def.outputs[output_key].name
with tf.Session() as sess:
loader.load(sess, [_TAG], saved_model_dir)
outputs = sess.run(output_tensor, feed_dict=inputs_feed_dict)
return outputs
def testPredictionSignatureDef(self):
input1 = constant_op.constant("a", name="input-1")
input2 = constant_op.constant("b", name="input-2")
output1 = constant_op.constant("c", name="output-1")
output2 = constant_op.constant("d", name="output-2")
meta_graph_def = meta_graph_pb2.MetaGraphDef()
self._add_to_signature_def_map(
meta_graph_def, {
"my_prediction":
signature_def_utils.predict_signature_def(
{
"input-1": input1,
"input-2": input2
}, {
"output-1": output1,
"output-2": output2
})
})
# Look up the prediction signature def with the key used while saving.
signature_def = signature_def_contrib_utils.get_signature_def_by_key(
meta_graph_def, "my_prediction")
self.assertEqual(signature_constants.PREDICT_METHOD_NAME,
signature_def.method_name)
# Check inputs in signature def.
self.assertEqual(2, len(signature_def.inputs))
self._check_tensor_info(signature_def.inputs, "input-1", "input-1:0")
self._check_tensor_info(signature_def.inputs, "input-2", "input-2:0")
# Check outputs in signature def.
self.assertEqual(2, len(signature_def.outputs))
self._check_tensor_info(signature_def.outputs, "output-1",
"output-1:0")
self._check_tensor_info(signature_def.outputs, "output-2",
"output-2:0")
def __init__(self, model_path):
self.logger = logging.getLogger(self.__class__.__name__)
self.model_path = model_path
if not tf.saved_model.loader.maybe_saved_model_directory(
self.model_path):
raise ValueError('No model found in', self.model_path)
self.sess = tf.Session(graph=tf.Graph())
meta_graph_def = tf.saved_model.loader.load(
self.sess, [tf.saved_model.tag_constants.SERVING], self.model_path)
signature_def = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
meta_graph_def_sig = signature_def_utils.get_signature_def_by_key(
meta_graph_def, signature_def)
self.input_tensor_info = meta_graph_def_sig.inputs
self.output_tensor_info = meta_graph_def_sig.outputs
self.output_tensor_keys = [k for k in self.output_tensor_info]
self.output_tensor_names = [
self.output_tensor_info[k].name for k in self.output_tensor_keys
]
def __init__(self, model_path, vocab_fname, output_fname, logger, skip_attention):
self.logger = logger
self.model_path = model_path
self.output_fname = output_fname
if not tf.saved_model.loader.maybe_saved_model_directory(self.model_path):
raise ValueError('No model found in', self.model_path)
self.sess = tf.Session(graph=tf.Graph())
meta_graph_def = tf.saved_model.loader.load(self.sess, [tf.saved_model.tag_constants.SERVING], self.model_path)
signature_def = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
meta_graph_def_sig = signature_def_utils.get_signature_def_by_key(meta_graph_def, signature_def)
input_tensor_keys = [k for k in meta_graph_def_sig.inputs]
input_tensor_names = [meta_graph_def_sig.inputs[k].name for k in input_tensor_keys]
self.t_variables = {key: name for key, name in zip(input_tensor_keys, input_tensor_names)}
self.final_output = meta_graph_def_sig.outputs['output'].name
self.str_scores = meta_graph_def_sig.outputs['str_scores'].name
self.vocab = utils.load_dict(vocab_fname)
def _from_saved_model_v1(sess, model_path, input_names, output_names, tag,
signature_names):
"""Load tensorflow graph from saved_model."""
wrn_no_tag = "'--tag' not specified for saved_model. Using --tag serve"
wrn_empty_tag = "'--tag' value is empty string. Using tags = []"
wrn_empty_sig = "'--signature_def' not provided. Using all signatures."
if tag is None:
tag = [tf.saved_model.tag_constants.SERVING]
logger.warning(wrn_no_tag)
if not signature_names:
logger.warning(wrn_empty_sig)
if tag == '':
tag = []
logger.warning(wrn_empty_tag)
if not isinstance(tag, list):
tag = [tag]
imported = tf.saved_model.loader.load(sess, tag, model_path)
signatures = []
for k in imported.signature_def.keys():
if k in signature_names or (not signature_names
and not k.startswith("_")):
signatures.append(k)
try:
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
# pylint: disable=unnecessary-lambda
get_signature_def = lambda meta_graph_def, k: \
signature_def_utils.get_signature_def_by_key(meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[
k]
if input_names is None:
input_names = []
for k in signatures:
inputs_tensor_info = get_signature_def(imported, k).inputs
for _, input_tensor in inputs_tensor_info.items():
if input_tensor.name not in input_names:
input_names.append(input_tensor.name)
tensors_to_rename = {}
if output_names is None:
output_names = []
for k in signatures:
outputs_tensor_info = get_signature_def(imported, k).outputs
for structured_name, output_tensor in outputs_tensor_info.items():
if output_tensor.name not in output_names:
output_names.append(output_tensor.name)
tensors_to_rename[output_tensor.name] = structured_name
frozen_graph = freeze_session(sess,
input_names=input_names,
output_names=output_names)
table_names, key_dtypes, value_dtypes = get_hash_table_info(frozen_graph)
initialized_tables = {}
tf.tables_initializer().run()
for n, k_dtype, val_dtype in zip(table_names, key_dtypes, value_dtypes):
h = lookup_ops.hash_table_v2(k_dtype, val_dtype, shared_name=n)
try:
k, v = lookup_ops.lookup_table_export_v2(h, k_dtype, val_dtype)
k, v = sess.run([k, v])
initialized_tables[n] = (k, v)
except Exception: # pylint: disable=broad-except
logger.warning("Could not initialize table with shared_name = %r",
n)
return frozen_graph, input_names, output_names, initialized_tables, tensors_to_rename
import pandas as pd
import tensorflow as tf
from tensorflow.python.tools import saved_model_utils
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
import common
tag = tf.saved_model.tag_constants.SERVING
signature_def = tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
with tf.Session() as sess:
export_dir = common.get_export_dir()
tf.saved_model.loader.load(sess, [tag], export_dir)
meta_graph_def = saved_model_utils.get_meta_graph_def(export_dir, tag)
predict_signature_def = signature_def_utils.get_signature_def_by_key(
meta_graph_def, signature_def)
inputs = common.get_test_inputs()
examples = common.create_examples(inputs)
fetches = [
predict_signature_def.outputs[key].name
for key in ['classes', 'scores']
]
feed_dict = {predict_signature_def.inputs['inputs'].name: examples}
outputs = sess.run(fetches, feed_dict=feed_dict)
predictions = {
'classes': outputs[0],
'scores': outputs[1],
}
def run_test(self,
name,
backend="caffe2",
debug=False,
onnx_file=None,
opset=None,
perf=None,
fold_const=None):
"""Run complete test against backend."""
print(name)
self.perf = perf
# get the model
if self.url:
_, dir_name = self.download_file()
model_path = os.path.join(dir_name, self.local)
else:
model_path = self.local
dir_name = os.path.dirname(self.local)
print("\tdownloaded", model_path)
if self.model_type in ["checkpoint"]:
#
# if the input model is a checkpoint, convert it to a frozen model
saver = tf.train.import_meta_graph(model_path)
with tf.Session() as sess:
saver.restore(sess, model_path[:-5])
frozen_graph = freeze_session(sess,
output_names=self.output_names)
tf.train.write_graph(frozen_graph,
dir_name,
"frozen.pb",
as_text=False)
model_path = os.path.join(dir_name, "frozen.pb")
elif self.model_type in ["saved_model"]:
try:
from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
get_signature_def = lambda meta_graph_def, k: \
signature_def_utils.get_signature_def_by_key(meta_graph_def, k)
except ImportError:
# TF1.12 changed the api
get_signature_def = lambda meta_graph_def, k: meta_graph_def.signature_def[
k]
# saved_model format - convert to checkpoint
with tf.Session() as sess:
meta_graph_def = tf.saved_model.loader.load(
sess, [tf.saved_model.tag_constants.SERVING], model_path)
inputs = {}
outputs = {}
for k in meta_graph_def.signature_def.keys():
inputs_tensor_info = get_signature_def(meta_graph_def,
k).inputs
for _, input_tensor in sorted(inputs_tensor_info.items()):
inputs[
input_tensor.name] = sess.graph.get_tensor_by_name(
input_tensor.name)
outputs_tensor_info = get_signature_def(meta_graph_def,
k).outputs
for _, output_tensor in sorted(
outputs_tensor_info.items()):
outputs[output_tensor.
name] = sess.graph.get_tensor_by_name(
output_tensor.name)
# freeze uses the node name derived from output:0 so only pass in output:0;
# it will provide all outputs of that node.
for o in list(outputs.keys()):
if not o.endswith(":0"):
del outputs[o]
frozen_graph = freeze_session(sess,
output_names=list(
outputs.keys()))
tf.train.write_graph(frozen_graph,
dir_name,
"frozen.pb",
as_text=False)
model_path = os.path.join(dir_name, "frozen.pb")
# create the input data
inputs = {}
for k, v in self.input_names.items():
if isinstance(v, six.text_type) and v.startswith("np."):
inputs[k] = eval(v) # pylint: disable=eval-used
else:
inputs[k] = self.make_input(v)
if self.more_inputs:
for k, v in self.more_inputs.items():
inputs[k] = v
tf.reset_default_graph()
graph_def = graph_pb2.GraphDef()
with open(model_path, "rb") as f:
graph_def.ParseFromString(f.read())
graph_def = tf2onnx.tfonnx.tf_optimize(inputs.keys(),
self.output_names, graph_def,
fold_const)
shape_override = {}
g = tf.import_graph_def(graph_def, name='')
with tf.Session(config=tf.ConfigProto(allow_soft_placement=True),
graph=g) as sess:
# fix inputs if needed
for k in inputs.keys(): # pylint: disable=consider-iterating-dictionary
t = sess.graph.get_tensor_by_name(k)
dtype = tf.as_dtype(t.dtype).name
if type != "float32":
v = inputs[k]
inputs[k] = v.astype(dtype)
if self.force_input_shape:
for k, v in inputs.items():
shape_override[k] = list(v.shape)
# run the model with tensorflow
if self.skip_tensorflow:
print("\ttensorflow", "SKIPPED")
else:
tf_results = self.run_tensorflow(sess, inputs)
print("\ttensorflow", "OK")
model_proto = None
try:
# convert model to onnx
onnx_graph = self.to_onnx(sess.graph,
opset=opset,
shape_override=shape_override,
input_names=inputs.keys())
new_model_proto = GraphUtil.opt_transposes_with_graph(
onnx_graph, "test", debug=debug)
if new_model_proto:
model_proto = new_model_proto
else:
print(
"\tNON-CRITICAL, optimizers are not applied successfully"
)
print("\tto_onnx", "OK")
if debug:
onnx_graph.dump_graph()
if onnx_file:
self.create_onnx_file(name, model_proto, inputs, onnx_file)
except Exception as ex:
tb = traceback.format_exc()
print("\tto_onnx", "FAIL", ex, tb)
try:
onnx_results = None
if backend == "caffe2":
onnx_results = self.run_caffe2(name, model_proto, inputs)
elif backend == "onnxmsrtnext":
onnx_results = self.run_onnxmsrtnext(name, model_proto, inputs)
elif backend == "onnxruntime":
onnx_results = self.run_onnxruntime(name, model_proto, inputs)
else:
raise ValueError("unknown backend")
print("\trun_onnx OK")
try:
if self.skip_tensorflow:
print("\tResults: skipped tensorflow")
else:
if self.check_only_shape:
for tf_res, onnx_res in zip(tf_results, onnx_results):
np.testing.assert_array_equal(
tf_res.shape, onnx_res.shape)
else:
for tf_res, onnx_res in zip(tf_results, onnx_results):
np.testing.assert_allclose(tf_res,
onnx_res,
rtol=self.rtol,
atol=self.atol)
print("\tResults: OK")
return True
except Exception as ex:
print("\tResults: ", ex)
except Exception as ex:
print("\trun_onnx", "FAIL", ex)
return False
def _run_model(iterator, args):
"""Run single-node inferencing on a checkpoint/saved_model using input tensors obtained from a Spark partition iterator and returning output tensors."""
single_node_env(args)
logging.info("===== input_mapping: {}".format(args.input_mapping))
logging.info("===== output_mapping: {}".format(args.output_mapping))
input_tensor_names = [ tensor for col,tensor in sorted(args.input_mapping.items()) ]
output_tensor_names = [ tensor for tensor,col in sorted(args.output_mapping.items()) ]
# if using a signature_def_key, get input/output tensor info from the requested signature
if args.signature_def_key:
assert args.export_dir, "Inferencing with signature_def_key requires --export_dir argument"
logging.info("===== loading meta_graph_def for tag_set ({0}) from saved_model: {1}".format(args.tag_set, args.export_dir))
meta_graph_def = get_meta_graph_def(args.export_dir, args.tag_set)
signature = signature_def_utils.get_signature_def_by_key(meta_graph_def, args.signature_def_key)
logging.info("signature: {}".format(signature))
inputs_tensor_info = signature.inputs
logging.info("inputs_tensor_info: {0}".format(inputs_tensor_info))
outputs_tensor_info = signature.outputs
logging.info("outputs_tensor_info: {0}".format(outputs_tensor_info))
result = []
with tf.Session(graph=ops_lib.Graph()) as sess:
if args.export_dir:
assert args.tag_set, "Inferencing from a saved_model requires --tag_set"
# load graph from a saved_model
logging.info("===== restoring from saved_model: {}".format(args.export_dir))
loader.load(sess, args.tag_set.split(','), args.export_dir)
elif args.model_dir:
# load graph from a checkpoint
ckpt = tf.train.latest_checkpoint(args.model_dir)
assert ckpt, "Invalid model checkpoint path: {}".format(args.model_dir)
logging.info("===== restoring from checkpoint: {}".format(ckpt + ".meta"))
saver = tf.train.import_meta_graph(ckpt + ".meta", clear_devices=True)
saver.restore(sess, ckpt)
else:
raise Exception("Inferencing requires either --model_dir or --export_dir argument")
# get list of input/output tensors (by name)
if args.signature_def_key:
input_tensors = [inputs_tensor_info[t].name for t in input_tensor_names]
output_tensors = [outputs_tensor_info[output_tensor_names[0]].name]
else:
input_tensors = [t + ':0' for t in input_tensor_names]
output_tensors = [t + ':0' for t in output_tensor_names]
logging.info("input_tensors: {0}".format(input_tensors))
logging.info("output_tensors: {0}".format(output_tensors))
# feed data in batches and return output tensors
for tensors in yield_batch(iterator, args.batch_size, len(input_tensor_names)):
inputs_feed_dict = {}
for i in range(len(input_tensors)):
inputs_feed_dict[input_tensors[i]] = tensors[i]
outputs = sess.run(output_tensors, feed_dict=inputs_feed_dict)
lengths = [ len(output) for output in outputs ]
input_size = len(tensors[0])
assert all([ l == input_size for l in lengths ]), "Output array sizes {} must match input size: {}".format(lengths, input_size)
python_outputs = [ output.tolist() for output in outputs ] # convert from numpy to standard python types
result.extend(zip(*python_outputs)) # convert to an array of tuples of "output columns"
return result
def run_test(self, name, backend="caffe2", debug=False, onnx_file=None, opset=None, perf=None, fold_const=None):
"""Run complete test against backend."""
print(name)
self.perf = perf
# get the model
if self.url:
_, dir_name = self.download_file()
model_path = os.path.join(dir_name, self.local)
else:
model_path = self.local
dir_name = os.path.dirname(self.local)
print("\tdownloaded", model_path)
if self.model_type in ["checkpoint"]:
#
# if the input model is a checkpoint, convert it to a frozen model
saver = tf.train.import_meta_graph(model_path)
with tf.Session() as sess:
saver.restore(sess, model_path[:-5])
frozen_graph = freeze_session(sess, output_names=self.output_names)
tf.train.write_graph(frozen_graph, dir_name, "frozen.pb", as_text=False)
model_path = os.path.join(dir_name, "frozen.pb")
elif self.model_type in ["saved_model"]:
# saved_model format - convert to checkpoint
with tf.Session() as sess:
meta_graph_def = tf.saved_model.loader.load(sess, [tf.saved_model.tag_constants.SERVING], model_path)
inputs = {}
outputs = {}
for k in meta_graph_def.signature_def.keys():
inputs_tensor_info = signature_def_utils.get_signature_def_by_key(meta_graph_def, k).inputs
for _, input_tensor in sorted(inputs_tensor_info.items()):
inputs[input_tensor.name] = sess.graph.get_tensor_by_name(input_tensor.name)
outputs_tensor_info = signature_def_utils.get_signature_def_by_key(meta_graph_def, k).outputs
for _, output_tensor in sorted(outputs_tensor_info.items()):
outputs[output_tensor.name] = sess.graph.get_tensor_by_name(output_tensor.name)
frozen_graph = freeze_session(sess, output_names=list(outputs.keys()))
tf.train.write_graph(frozen_graph, dir_name, "frozen.pb", as_text=False)
model_path = os.path.join(dir_name, "frozen.pb")
# create the input data
inputs = {}
for k, v in self.input_names.items():
if isinstance(v, six.text_type) and v.startswith("np."):
inputs[k] = eval(v) # pylint: disable=eval-used
else:
inputs[k] = self.make_input(v)
if self.more_inputs:
for k, v in self.more_inputs.items():
inputs[k] = v
tf.reset_default_graph()
graph_def = graph_pb2.GraphDef()
with open(model_path, "rb") as f:
graph_def.ParseFromString(f.read())
graph_def = tf2onnx.tfonnx.tf_optimize(inputs, self.output_names, graph_def, fold_const)
shape_override = {}
g = tf.import_graph_def(graph_def, name='')
with tf.Session(graph=g) as sess:
# fix inputs if needed
for k in inputs.keys(): # pylint: disable=consider-iterating-dictionary
t = sess.graph.get_tensor_by_name(k)
dtype = tf.as_dtype(t.dtype).name
if type != "float32":
v = inputs[k]
inputs[k] = v.astype(dtype)
if self.force_input_shape:
shape_override = self.input_names
# run the model with tensorflow
if self.skip_tensorflow:
print("\ttensorflow", "SKIPPED")
else:
tf_results = self.run_tensorflow(sess, inputs)
print("\ttensorflow", "OK")
model_proto = None
try:
# convert model to onnx
onnx_graph = self.to_onnx(sess.graph, opset=opset, shape_override=shape_override)
optimizer = TransposeOptimizer(onnx_graph, self.output_names, debug)
optimizer.optimize()
model_proto = onnx_graph.make_model("test")
print("\tto_onnx", "OK")
if debug:
onnx_graph.dump_graph()
if onnx_file:
self.create_onnx_file(name, model_proto, inputs, onnx_file)
except Exception as ex:
print("\tto_onnx", "FAIL", ex)
try:
onnx_results = None
if backend == "caffe2":
onnx_results = self.run_caffe2(name, model_proto, inputs)
elif backend == "onnxmsrtnext":
onnx_results = self.run_onnxmsrtnext(name, model_proto, inputs)
elif backend == "onnxruntime":
onnx_results = self.run_onnxruntime(name, model_proto, inputs)
else:
raise ValueError("unknown backend")
print("\trun_onnx OK")
try:
if self.skip_tensorflow:
print("\tResults: skipped tensorflow")
else:
if self.check_only_shape:
for tf_res, onnx_res in zip(tf_results, onnx_results):
np.testing.assert_array_equal(tf_res.shape, onnx_res.shape)
else:
for tf_res, onnx_res in zip(tf_results, onnx_results):
np.testing.assert_allclose(tf_res, onnx_res, rtol=self.rtol, atol=self.atol)
print("\tResults: OK")
return True
except Exception as ex:
print("\tResults: ", ex)
except Exception as ex:
print("\trun_onnx", "FAIL", ex)
return False
def _run_model(iterator, args, tf_args):
"""mapPartitions function to run single-node inferencing from a checkpoint/saved_model, using the model's input/output mappings.
Args:
:iterator: input RDD partition iterator.
:args: arguments for TFModel, in argparse format
:tf_args: arguments for TensorFlow inferencing code, in argparse or ARGV format.
Returns:
An iterator of result data.
"""
single_node_env(tf_args)
logging.info("===== input_mapping: {}".format(args.input_mapping))
logging.info("===== output_mapping: {}".format(args.output_mapping))
input_tensor_names = [
tensor for col, tensor in sorted(args.input_mapping.items())
]
output_tensor_names = [
tensor for tensor, col in sorted(args.output_mapping.items())
]
# if using a signature_def_key, get input/output tensor info from the requested signature
if args.signature_def_key:
assert args.export_dir, "Inferencing with signature_def_key requires --export_dir argument"
logging.info(
"===== loading meta_graph_def for tag_set ({0}) from saved_model: {1}"
.format(args.tag_set, args.export_dir))
meta_graph_def = get_meta_graph_def(args.export_dir, args.tag_set)
signature = signature_def_utils.get_signature_def_by_key(
meta_graph_def, args.signature_def_key)
logging.debug("signature: {}".format(signature))
inputs_tensor_info = signature.inputs
logging.debug("inputs_tensor_info: {0}".format(inputs_tensor_info))
outputs_tensor_info = signature.outputs
logging.debug("outputs_tensor_info: {0}".format(outputs_tensor_info))
result = []
global global_sess, global_args
if global_sess and global_args == args:
# if graph/session already loaded/started (and using same args), just reuse it
sess = global_sess
else:
# otherwise, create new session and load graph from disk
tf.reset_default_graph()
sess = tf.Session(graph=tf.get_default_graph())
if args.export_dir:
assert args.tag_set, "Inferencing from a saved_model requires --tag_set"
# load graph from a saved_model
logging.info("===== restoring from saved_model: {}".format(
args.export_dir))
loader.load(sess, args.tag_set.split(','), args.export_dir)
elif args.model_dir:
# load graph from a checkpoint
ckpt = tf.train.latest_checkpoint(args.model_dir)
assert ckpt, "Invalid model checkpoint path: {}".format(
args.model_dir)
logging.info("===== restoring from checkpoint: {}".format(ckpt +
".meta"))
saver = tf.train.import_meta_graph(ckpt + ".meta",
clear_devices=True)
saver.restore(sess, ckpt)
else:
raise Exception(
"Inferencing requires either --model_dir or --export_dir argument"
)
global_sess = sess
global_args = args
# get list of input/output tensors (by name)
if args.signature_def_key:
input_tensors = [
inputs_tensor_info[t].name for t in input_tensor_names
]
output_tensors = [outputs_tensor_info[output_tensor_names[0]].name]
else:
input_tensors = [t + ':0' for t in input_tensor_names]
output_tensors = [t + ':0' for t in output_tensor_names]
logging.info("input_tensors: {0}".format(input_tensors))
logging.info("output_tensors: {0}".format(output_tensors))
# feed data in batches and return output tensors
for tensors in yield_batch(iterator, args.batch_size,
len(input_tensor_names)):
inputs_feed_dict = {}
for i in range(len(input_tensors)):
inputs_feed_dict[input_tensors[i]] = tensors[i]
outputs = sess.run(output_tensors, feed_dict=inputs_feed_dict)
lengths = [len(output) for output in outputs]
input_size = len(tensors[0])
assert all([
length == input_size for length in lengths
]), "Output array sizes {} must match input size: {}".format(
lengths, input_size)
python_outputs = [output.tolist() for output in outputs
] # convert from numpy to standard python types
result.extend(zip(*python_outputs)
) # convert to an array of tuples of "output columns"
return result
def get_meta_graph_def(saved_model_dir, tags):
"""Gets `MetaGraphDef` from a directory containing a `SavedModel`.
Returns the `MetaGraphDef` for the given tag-set and SavedModel directory.
Args:
saved_model_dir: Directory containing the SavedModel.
tags: Comma separated list of tags used to identify the correct
`MetaGraphDef`.
Raises:
ValueError: An error when the given tags cannot be found.
Returns:
A `MetaGraphDef` corresponding to the given tags.
"""
saved_model = reader.read_saved_model(saved_model_dir)
set_of_tags = set([tag.strip() for tag in tags.split(',')])
for meta_graph_def in saved_model.meta_graphs:
if set(meta_graph_def.meta_info_def.tags) == set_of_tags:
return meta_graph_def
raise ValueError('Could not find MetaGraphDef with tags {}'.format(tags))
exp_dir = "/home/heavens/Chiron_project/Chiron/chiron/model/DNA_default/export/1/"
metagraph_def = get_meta_graph_def(exp_dir, 'serve')
signature_def = signature_def_utils.get_signature_def_by_key(
metagraph_def, 'predicted_sequences')
output_names = {k: v.name for k, v in signature_def.outputs.items()}
print(output_names)
predict_fn = predictor.from_saved_model(exp_dir)
seq_len = np.asarray([[4]])
combined_input = np.concatenate((batch_x, seq_len), axis=1)
predictions = predict_fn({'inputs': combined_input})
