def testApiDefMapPutThenGet(self):
api_def_map = c_api_util.ApiDefMap()
api_def_text = """
op {
graph_op_name: "Add"
summary: "Returns x + y element-wise."
description: <<END
*NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
[here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
END
}
"""
api_def_map.put_api_def(api_def_text)
api_def = api_def_map.get_api_def("Add")
self.assertEqual(api_def.graph_op_name, "Add")
self.assertEqual(api_def.summary, "Returns x + y element-wise.")
def main(argv):
del argv # Unused.
if FLAGS.output_path is None:
raise ValueError('no output_path has been set')
api_def_map = c_api_util.ApiDefMap()
op_codes = []
enum_store = EnumStore()
op_names = api_def_map.op_names()
if FLAGS.api_def_path is not None:
for op_name in op_names:
path = os.path.join(FLAGS.api_def_path, 'api_def_%s.pbtxt' % op_name)
if not tf.gfile.Exists(path):
continue
with tf.gfile.Open(path, 'r') as fobj:
data = fobj.read()
api_def_map.put_api_def(data)
for op_name in sorted(op_names):
try:
if op_name[0] == '_': continue
op = api_def_map.get_op_def(op_name)
api_def = api_def_map.get_api_def(bytes(op_name))
op_codes.append(generate_code(op, api_def, enum_store))
except UnableToGenerateCodeError as e:
print('Cannot generate code for %s: %s' % (op.name, e.details))
print('Generated code for %d/%d ops.' % (len(op_codes), len(op_names)))
version_codes = [
'static let generatedTensorFlowVersion = "%s"' % tf.__version__,
'static let generatedTensorFlowGitVersion = "%s"' % tf.__git_version__,
]
swift_code = (
_WARNING +
_HEADER +
'\npublic enum Raw {\n\n' +
'\n'.join(version_codes) +
'\n\n' +
'\n\n'.join(enum_store.enum_codes()) +
'\n\n' +
'\n\n'.join(op_codes) +
'\n\n}')
with tf.gfile.Open(FLAGS.output_path, 'w') as fobj:
fobj.write(swift_code)
import logging
from onnx import numpy_helper, helper
import numpy as np
from google.protobuf.json_format import ParseDict
import tensorflow as tf
from tensorflow.python.framework import c_api_util
from tensorflow.core.framework import types_pb2, node_def_pb2
from tf2onnx import utils
from tf2onnx.graph import Graph
from tf2onnx import tf_utils
logger = logging.getLogger(__name__)
tf_api_def_map = c_api_util.ApiDefMap()
def read_tfjs_attr(attr, tf_dtypes=False):
"""
Reads the value of a single tfjs node attribute. If tf_dtypes is True, tensorflow dtypes are returned instead of
onnx dtypes
"""
k = list(attr.keys())[0]
return read_tfjs_attr_helper(k, attr[k], tf_dtypes)
def fix_string_attr(tfjs_node):
"""
Older tfjs models store strings as lists of ints (representing byte values). This function finds and replaces
those strings, so protobuf can correctly decode the json.
def testApiDefMapGet(self):
api_def_map = c_api_util.ApiDefMap()
op_def = api_def_map.get_op_def("Add")
self.assertEqual(op_def.name, "Add")
api_def = api_def_map.get_api_def("Add")
self.assertEqual(api_def.graph_op_name, "Add")
def main(argv):
del argv # Unused.
if FLAGS.output_path is None:
raise ValueError('No output_path has been set')
api_def_map = c_api_util.ApiDefMap()
op_codes = []
op_codes_forwarding = []
enum_store = EnumStore()
op_names = api_def_map.op_names()
if FLAGS.api_def_path is not None:
for op_name in op_names:
path = os.path.join(FLAGS.api_def_path,
'api_def_%s.pbtxt' % op_name)
if not tf.gfile.Exists(path):
continue
with tf.gfile.Open(path, 'r') as fobj:
data = fobj.read()
try:
api_def_map.put_api_def(data)
except Exception as e:
print('Cannot load api def for %s: %s' % (op_name, str(e)))
num_generated = 0
for op_name in sorted(op_names):
try:
if op_name[0] == '_': continue
op_def = api_def_map.get_op_def(op_name)
if any(a.is_ref for a in op_def.input_arg):
raise UnableToGenerateCodeError('has ref-valued input')
if any(a.is_ref for a in op_def.output_arg):
raise UnableToGenerateCodeError('has ref-valued output')
api_def = api_def_map.get_api_def(bytes(op_name, 'utf8'))
# It would be nicer to handle `StringTensor` in a more
# general way by having `String` conform to `TensorFlowScalar`.
default_op = Op(op_def, api_def, enum_store, string_valued=False)
string_valued_op = Op(op_def,
api_def,
enum_store,
string_valued=True)
default_code = default_op.swift_function()
string_valued_code = string_valued_op.swift_function()
op_codes.append(default_code)
string_valued_op_different = False
if string_valued_code != default_code:
string_valued_op_different = True
op_codes.append(string_valued_code)
default_code = default_op.swift_dispatch_function(
x10_supported=op_name in X10_OPS)
string_valued_code = string_valued_op.swift_dispatch_function()
op_codes_forwarding.append(default_code)
if string_valued_op_different:
op_codes_forwarding.append(string_valued_code)
num_generated += 1
except UnableToGenerateCodeError as e:
print('Cannot generate code for %s: %s' % (op_name, e.details))
print('Generated code for %d/%d ops.' % (num_generated, len(op_names)))
version_codes = [
' static let generatedTensorFlowVersion = "%s"' % tf.__version__,
' static let generatedTensorFlowGitVersion = "%s"' %
tf.__git_version__
]
swift_code = (
_WARNING + _HEADER + 'import CTensorFlow\n\n' +
'@inlinable @inline(__always)\n' +
'func makeOp(_ name: String, _ nOutputs: Int) -> TFTensorOperation {\n'
+ ' _ExecutionContext.makeOp(name, nOutputs)\n' + '}\n' +
'\n\npublic enum _RawTFEager {\n\n' + '\n'.join(version_codes) +
'\n\n' + '\n\n'.join(enum_store.enum_codes()) + '\n\n' +
'\n'.join(op_codes) + '\n\n}\n')
with tf.gfile.Open(FLAGS.output_path, 'w') as f:
f.write(swift_code)
swift_code = (_WARNING + _HEADER + _DISPATCHER_TEMPLATE.format(
raw_dispatching_enum='public enum _Raw {\n\n' +
'\n'.join(version_codes) + '\n\n' +
'\n\n'.join(enum_store.enum_codes_forwarding()) + '\n\n' +
'\n'.join(op_codes_forwarding) + '\n\n}'))
if FLAGS.dispatching_output_path:
with tf.gfile.Open(FLAGS.dispatching_output_path, 'w') as f:
f.write(swift_code)
def testApiDefMapOpNames(self):
api_def_map = c_api_util.ApiDefMap()
self.assertIn("Add", api_def_map.op_names())
def main(argv):
del argv # Unused.
if FLAGS.output_path is None:
raise ValueError('No output_path has been set')
api_def_map = c_api_util.ApiDefMap()
op_codes = []
enum_store = EnumStore()
op_names = api_def_map.op_names()
if FLAGS.api_def_path is not None:
for op_name in op_names:
path = os.path.join(FLAGS.api_def_path, 'api_def_%s.pbtxt' % op_name)
if not tf.gfile.Exists(path):
continue
with tf.gfile.Open(path, 'r') as fobj:
data = fobj.read()
try:
api_def_map.put_api_def(data)
except Exception as e:
print('Cannot load api def for %s: %s' % (op_name, str(e)))
num_generated = 0
for op_name in sorted(op_names):
try:
if op_name[0] == '_': continue
op_def = api_def_map.get_op_def(op_name)
if any(a.is_ref for a in op_def.input_arg):
raise UnableToGenerateCodeError('has ref-valued input')
if any(a.is_ref for a in op_def.output_arg):
raise UnableToGenerateCodeError('has ref-valued output')
api_def = api_def_map.get_api_def(bytes(op_name, 'utf8'))
# It would be nicer to handle `StringTensor` in a more
# general way by having `String` conform to `TensorFlowScalar`.
default_op = Op(op_def, api_def, enum_store, string_valued=False)
string_valued_op = Op(op_def, api_def, enum_store, string_valued=True)
default_code = default_op.swift_function()
string_valued_code = string_valued_op.swift_function()
op_codes.append(default_code)
if string_valued_code != default_code:
op_codes.append(string_valued_code)
num_generated += 1
except UnableToGenerateCodeError as e:
print('Cannot generate code for %s: %s' % (op_name, e.details))
print('Generated code for %d/%d ops.' % (num_generated, len(op_names)))
version_codes = [
'static let generatedTensorFlowVersion = "%s"' % tf.__version__,
'static let generatedTensorFlowGitVersion = "%s"' % tf.__git_version__]
swift_code = (
_WARNING +
_HEADER +
'import CTensorFlow\n\n' +
'\npublic enum Raw {\n\n' +
'\n'.join(version_codes) +
'\n\n' +
'\n\n'.join(enum_store.enum_codes()) +
'\n\n' +
'\n'.join(op_codes) +
'\n\n}\n')
with tf.gfile.Open(FLAGS.output_path, 'w') as f:
f.write(swift_code)
