def test_forward_inception_v1():
'''test inception V1 model'''
with tf.Graph().as_default():
graph_def = tf_testing.get_workload("InceptionV1/classify_image_graph_def-with_shapes.pb")
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# Build an image from random data.
from PIL import Image
from tvm.contrib import util
img_array = np.random.uniform(size=(1, 600, 600, 3)).astype("uint8")
img = Image.frombuffer('RGB', (600, 600), img_array.tostring(), 'raw', 'RGB', 0, 1)
temp = util.tempdir()
img_path = temp.relpath("tf-test.jpg")
img.save(img_path);
import os.path
if not tf.gfile.Exists(os.path.join(img_path)):
tf.logging.fatal('File does not exist %s', image)
data = tf.gfile.FastGFile(os.path.join(img_path), 'rb').read()
temp.remove()
# Extract tensorflow decoded image frame for tvm input
with tf.Session() as sess:
tvm_data = run_tf_graph(sess, data, 'DecodeJpeg/contents:0', 'DecodeJpeg:0')
with tf.Session() as sess:
tf_output = run_tf_graph(sess, data, 'DecodeJpeg/contents:0', 'softmax:0')
tvm_output = run_tvm_graph(graph_def, tvm_data, 'DecodeJpeg/contents')
tvm.testing.assert_allclose(tf_output[0], tvm_output[0], rtol=1e-5, atol=1e-5)
def from_tensorflow(model_path, shapes, outputs=None, opt_model_path=None):
# type: (str, dict, List[str, ]str) -> relay.expr.Module, dict
""" Load tensorflow model from file and convert to relay expression
"""
try:
import tensorflow as tf
except ImportError as e:
raise ImportError("Please install tensorflow before trying to import"
" tensorflow models.")
if outputs is None or len(outputs) < 1:
raise ValueError("Please provide the output names for the provided"
" Tensorflow model")
with tf.gfile.GFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
mod, params = relay.frontend.from_tensorflow(graph_def,
layout=None,
shape=shapes,
outputs=outputs)
# data_layout = 'NHWC'
# TODO: Get the right layout??
return mod, params
def create_graph():
"""Creates a graph from saved GraphDef file and returns a saver."""
# Creates graph from saved graph_def.pb.
with tf.gfile.FastGFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
def from_tensorflow(model_path, img_shape):
import tensorflow as tf
import tvm.relay.testing.tf as tf_testing
def analyze_tf_inputs_outputs(graph):
ops = graph.get_operations()
outputs_set = set(ops)
inputs = []
for op in ops:
if len(op.inputs) == 0 and op.type != 'Const':
inputs.append(op)
else:
for input_tensor in op.inputs:
if input_tensor.op in outputs_set:
outputs_set.remove(input_tensor.op)
outputs = [op.name for op in outputs_set]
outputs.sort()
inputs = [op.name for op in inputs]
return inputs, outputs
shape_dict = {}
with tf.compat.v1.gfile.GFile(model_path, 'rb') as f:
graph_def = tf.compat.v1.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
graph_inputs, graph_outputs = analyze_tf_inputs_outputs(tf.get_default_graph())
for node in graph_def.node:
if node.name in graph_inputs:
get_shape = ()
input_lenth = len(node.attr['shape'].shape.dim)
if input_lenth:
for i in range(input_lenth):
dim_size = node.attr['shape'].shape.dim[i].size
dim_size = dim_size if dim_size != -1 else None
get_shape += (dim_size,)
if len(get_shape) == 0 or not all(get_shape[1:]):
input_shape = (1,) + tuple(img_shape) + (3,)
elif not all(get_shape) and all(get_shape[1:]):
input_shape = (1,) + get_shape[1:]
else:
input_shape = get_shape
shape_dict[node.name] = input_shape
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# Add shapes to the graph.
with tf.compat.v1.Session() as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, graph_outputs)
mod, params = relay.frontend.from_tensorflow(graph_def, layout=None, shape=shape_dict)
Model = namedtuple('Model', ['mod', 'params', 'graph_inputs'])
model = Model(mod, params, graph_inputs)
return model
def test_forward_inception_v3():
'''test inception V3 model'''
with tf.Graph().as_default():
graph_def = tf_testing.get_workload('InceptionV3/inception_v3_2016_08_28_frozen-with_shapes.pb')
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
data = np.random.uniform(size=(1, 299, 299, 3)).astype('float32')
with tf.Session() as sess:
tf_output = run_tf_graph(sess, data, 'input:0', 'InceptionV3/Predictions/Reshape_1:0')
tvm_output = run_tvm_graph(graph_def, data, 'input')
tvm.testing.assert_allclose(tf_output[0], tvm_output[0], rtol=1e-5, atol=1e-5)
def load(self, path, shape_dict=None, **kwargs):
# pylint: disable=C0415
import tensorflow as tf
import tvm.relay.testing.tf as tf_testing
with tf.io.gfile.GFile(path, "rb") as tf_graph:
content = tf_graph.read()
graph_def = tf.compat.v1.GraphDef()
graph_def.ParseFromString(content)
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
logger.debug("parse TensorFlow model and convert into Relay computation graph")
return relay.frontend.from_tensorflow(graph_def, shape=shape_dict, **kwargs)
def load(self, path):
# pylint: disable=C0415
import tensorflow as tf
import tvm.relay.testing.tf as tf_testing
with tf.io.gfile.GFile(path, "rb") as tf_graph:
content = tf_graph.read()
graph_def = tf.compat.v1.GraphDef()
graph_def.ParseFromString(content)
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
logging.debug("relay.frontend.from_tensorflow")
return relay.frontend.from_tensorflow(graph_def)
def importTFModel(model_path, output_name):
with tf.compat.v1.gfile.GFile(model_path, 'rb') as f:
graph_def = tf.compat.v1.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# Add shapes to the graph.
with tf.compat.v1.Session() as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, output_name)
return graph_def
def test_forward_resnetv2():
'''test resnet model'''
if is_gpu_available():
with tf.Graph().as_default():
graph_def = tf_testing.get_workload("ResnetV2/resnet-20180601_resnet_v2_imagenet-shapes.pb")
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
data = np.random.uniform(size=(128, 224, 224, 3)).astype('float32')
out_node = 'ArgMax'
with tf.Session() as sess:
tf_output = run_tf_graph(sess, data, 'input_tensor:0', out_node + ':0')
tvm_output = run_tvm_graph(graph_def, data, 'input_tensor', tf_output.shape, 'float32')
tvm.testing.assert_allclose(np.squeeze(tvm_output[0]), np.squeeze(tf_output[0]), rtol=1e-5, atol=1e-5)
def load(self, model_path, inputs=None, outputs=None):
# there is no input/output meta data i the graph so it need to come from config.
if not inputs:
raise ValueError("TVM needs inputs")
if not outputs:
raise ValueError("TVM needs outputs")
self.outputs = outputs
self.inputs = inputs
# Import model
# ------------
# Creates tensorflow graph definition from protobuf file.
with tf_compat_v1.gfile.GFile(model_path, "rb") as f:
graph_def = tf_compat_v1.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name="")
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# Add shapes to the graph.
with tf_compat_v1.Session() as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, "softmax")
# Import the graph to Relay
# -------------------------
# Import tensorflow graph definition to relay frontend.
mod, params = relay.frontend.from_tensorflow(graph_def,
layout=self.layout,
shape=None,
outputs=None)
# Relay Build
# -----------
# Compile the graph to llvm target with given input specification.
with tvm.transform.PassContext(opt_level=3):
lib = relay.build(mod,
target=self.target,
target_host=self.target_host,
params=params)
self.m = graph_runtime.GraphModule(lib["default"](self.ctx))
return self
def tf_tvm_lstm(opt_level):
with tf.Session() as sess:
inputs = model_inputs()
model = get_tf_model()
sess.run(tf.global_variables_initializer())
# saver = tf.compat.v1.train.Saver()
# saver.save(sess, model_path + "/tf_model")
# print([n.name for n in tf.get_default_graph().as_graph_def().node])
from util import export_pb
export_pb(sess,
model_path + "/tf_model.pb",
inputs=['Placeholder', 'Placeholder_1'],
outputs=["rnn/transpose_1"])
# _ = model.predict(inputs) # way to set input shape or it cannot be saved
# model.save(model_path + "/tf_mode.pb")
# tf.saved_model.save(model, model_path)
print("tvm compiling ...")
tic = time.time()
import tvm.relay.testing.tf as tf_testing
with tf.gfile.GFile(model_path + "/tf_model.pb", 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# Add shapes to the graph.
# with tf.Session() as sess:
# graph_def = tf_testing.AddShapesToGraphDef(sess, 'softmax')
# mod, params = relay.frontend.from_keras(model, shape=data_shape)
mod, params = relay.frontend.from_tensorflow(
graph_def,
shape={
"Placeholder": data_shape,
"Placeholder_1": [6, 2, data_shape[0], 2]
})
with relay.build_config(opt_level=3):
graph, lib, params = relay.build(mod,
target="llvm -mcpu=core-avx2")
tic = time.time() - tic
print("tvm compiling completed. spent %f seconds" % tic)
def test_forward_mobilenet():
'''test mobilenet model'''
# MobilenetV2
with tf.Graph().as_default():
graph_def = tf_testing.get_workload(
"https://storage.googleapis.com/mobilenet_v2/checkpoints/mobilenet_v2_1.4_224.tgz",
"mobilenet_v2_1.4_224_frozen.pb")
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
data = np.random.uniform(size=(1, 224, 224, 3)).astype('float32')
out_node = 'MobilenetV2/Predictions/Reshape_1'
with tf.Session() as sess:
# Add shapes to the graph.
graph_def = tf_testing.AddShapesToGraphDef(sess, out_node)
tf_output = run_tf_graph(sess, data, 'input:0', out_node + ':0')
tvm_output = run_tvm_graph(graph_def, data, 'input')
tvm.testing.assert_allclose(np.squeeze(tvm_output[0]), np.squeeze(tf_output[0]), rtol=1e-5, atol=1e-5)
def get_tf_model_InceptionV1(model_path):
with tf_compat_v1.gfile.GFile(model_path, 'rb') as f:
graph_def = tf_compat_v1.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# Add shapes to the graph.
with tf_compat_v1.Session() as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, 'softmax')
x = (299, 299, 3)
layout = None
shape_dict = {'DecodeJpeg/contents': x}
dtype_dict = {'DecodeJpeg/contents': 'uint8'}
mod, params = relay.frontend.from_tensorflow(
graph_def, layout="NCHW", shape={'DecodeJpeg/contents': (299, 299, 3)})
return mod, params, x
def export_tvm(weight_file='examples/brokenegg.npz', model_path=None):
import tvm.relay.testing.tf as tf_testing
from tvm import relay
import tvm
from brokenegg_transformer.runtime import transformer
graph, [inputs, targets], _ = transformer.load_model(weight_file,
as_graph=True)
with tf.Session(graph=graph) as sess:
graph_def = tf.graph_util.convert_variables_to_constants(
sess, graph.as_graph_def(), ["logits"])
if False:
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
with tf.Session(graph=graph) as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, 'logits')
layout = None
target = 'llvm'
target_host = 'llvm'
shape_dict = {'inputs': [None, None], 'targets': [None, None]}
dtype_dict = {
'inputs': "int64",
'targets': "int64",
}
with tf.Session(graph=graph) as sess:
mod, params = relay.frontend.from_tensorflow(graph_def,
layout=layout,
shape=shape_dict,
outputs=["logits"])
with tvm.transform.PassContext(opt_level=3):
graph, lib, params = relay.build(mod,
target=target,
target_host=target_host,
params=params)
def main():
# target setttings
target = 'llvm'
target_host = 'llvm'
layout = None
ctx = tvm.cpu(0)
model_path = argv[1]
graph_def = tf.GraphDef()
with tf.gfile.FastGFile(model_path, 'rb') as f:
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
shape_dict = {}
mod, params = relay.frontend.from_tensorflow(graph_def, layout=layout, shape=shape_dict)
print("Tensorflow protobuf imported to relay frontend.")
with relay.build_config(opt_level=3):
graph, lib, params = relay.build(mod, target=target, target_host=target_host, params=params)
def get_tf_yolov3_tiny(
model_path=("/hdd02/zhangyiyang/Tensorflow-YOLOv3/"
"weights/raw-yolov3-tiny.pb"),
outputs=['yolov3_tiny/concat_6'],):
input_shape = (1, 416, 416, 3)
with tf.compat.v1.gfile.GFile(model_path, 'rb') as f:
graph_def = tf.compat.v1.GraphDef()
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def, name='')
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
with tf.compat.v1.Session() as sess:
graph_def = tf_testing.AddShapesToGraphDef(
sess, outputs[0])
print("successfully load tf model")
mod, params = relay.frontend.from_tensorflow(
graph_def,
layout="NCHW",
shape={'Placeholder': input_shape},
outputs=outputs,
)
print("successfully convert tf model to relay")
return mod, params, input_shape
def run_timing(device, platform, model, remote=None, autotvm_log=None, batch=1, runs=3, reps=5, log=None):
"""
Run a time trail on TVM
:param device: The device to run this on
:param platform: The platform get the machine learning model on
:param model: The machine learning model to use
:param remote: Details about the remote device
:param autotvm_log: The path to the auto TVM file
:param batch: The number of pictures to run in one go
:param runs: The number of runs to run the picture through
:param reps: The number of times the measurement should be repeated
:param log: The output file
"""
# Output details of run
from cpuinfo import get_cpu_info
from datetime import datetime
print("\n──────────────────────────── TVMUI ────────────────────────────\n")
log.write("TVM Time Trial\n")
log_print(log, "Started on " + str(datetime.now().strftime("%m/%d/%Y at %H:%M:%S")))
if remote is None:
log_print(log, 'Hardware: ' + device)
if device == 'x86':
log_print(log, 'CPU Type: ' + get_cpu_info().get('brand_raw'))
else:
log_print(log, 'Remote Name: ' + remote["name"])
log_print(log, 'Remote Device: ' + remote["type"])
log_print(log, 'Remote Hardware: ' + remote["hardware"])
log_print(log, 'Backend: ' + platform)
log_print(log, 'Model: ' + model)
log_print(log, str(batch) + " picture(s) per run")
log_print(log, str(runs) + " run average, repeated " + str(reps) + " times.")
if autotvm_log is None:
log_print(log, 'AutoTVM: No\n')
else:
log_print(log, 'AutoTVM: Yes\n')
# Get the model and image data
import numpy as np
from PIL import Image
from tvm import relay
import tvm
from tvm.contrib.download import download_testdata
print("Loading models and images...")
pictures = get_pics(batch)
dataset = []
if platform == "MXNet":
from mxnet.gluon.model_zoo.vision import get_model
block = get_model(model, pretrained=True)
synset_url = "".join(
[
"https://gist.githubusercontent.com/zhreshold/",
"4d0b62f3d01426887599d4f7ede23ee5/raw/",
"596b27d23537e5a1b5751d2b0481ef172f58b539/",
"imagenet1000_clsid_to_human.txt",
]
)
synset_name = "imagenet1000_clsid_to_human.txt"
synset_path = download_testdata(synset_url, synset_name, module="data")
with open(synset_path) as f:
synset = eval(f.read())
def transform_image(image):
image = np.array(image) - np.array([123.0, 117.0, 104.0])
image /= np.array([58.395, 57.12, 57.375])
image = image.transpose((2, 0, 1))
image = image[np.newaxis, :]
return image
if model == 'resnet18_v1' or model == 'mobilenetv2_1.0':
for img in pictures:
dataset.append(transform_image(Image.open(img).resize((224, 224))))
input_shape = [batch, 3, 224, 224]
elif model == 'inceptionv3':
for img in pictures:
dataset.append(transform_image(Image.open(img).resize((299, 299))))
input_shape = [batch, 3, 299, 299]
else:
raise Exception("Invalid Model")
shape_dict = {"data": input_shape}
mod, params = relay.frontend.from_mxnet(block, shape_dict)
func = mod["main"]
func = relay.Function(func.params, relay.nn.softmax(func.body), None, func.type_params, func.attrs)
elif platform == "PyTorch":
import torch
import torchvision
model = getattr(torchvision.models, model)(pretrained=True)
model = model.eval()
# We grab the TorchScripted model via tracing
input_shape = [batch, 3, 224, 224]
input_data = torch.randn(input_shape)
scripted_model = torch.jit.trace(model, input_data).eval()
synset_url = "".join(
[
"https://raw.githubusercontent.com/Cadene/",
"pretrained-models.pytorch/master/data/",
"imagenet_synsets.txt",
]
)
synset_name = "imagenet_synsets.txt"
synset_path = download_testdata(synset_url, synset_name, module="data")
with open(synset_path) as f:
synsets = f.readlines()
synsets = [x.strip() for x in synsets]
splits = [line.split(" ") for line in synsets]
key_to_classname = {spl[0]: " ".join(spl[1:]) for spl in splits}
class_url = "".join(
[
"https://raw.githubusercontent.com/Cadene/",
"pretrained-models.pytorch/master/data/",
"imagenet_classes.txt",
]
)
class_name = "imagenet_classes.txt"
class_path = download_testdata(class_url, class_name, module="data")
with open(class_path) as f:
class_id_to_key = f.readlines()
class_id_to_key = [x.strip() for x in class_id_to_key]
def transform_image(image):
from torchvision import transforms
my_preprocess = transforms.Compose(
[
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
]
)
img = my_preprocess(image)
return np.expand_dims(img, 0)
for img in pictures:
dataset.append(transform_image(Image.open(img).resize((224, 224))))
input_name = "data"
shape_list = [(input_name, input_shape)]
func, params = relay.frontend.from_pytorch(scripted_model, shape_list)
elif platform == "TensorFlow":
import tensorflow as tf
import os
try:
tf_compat_v1 = tf.compat.v1
except ImportError:
tf_compat_v1 = tf
import tvm.relay.testing.tf as tf_testing
# Base location for model related files.
repo_base = "https://github.com/dmlc/web-data/raw/main/tensorflow/models/InceptionV1/"
model_name = "classify_image_graph_def-with_shapes.pb"
model_url = os.path.join(repo_base, model_name)
# Image label map
map_proto = "imagenet_2012_challenge_label_map_proto.pbtxt"
map_proto_url = os.path.join(repo_base, map_proto)
# Human readable text for labels
label_map = "imagenet_synset_to_human_label_map.txt"
label_map_url = os.path.join(repo_base, label_map)
model_path = download_testdata(model_url, model_name, module=["tf", "InceptionV1"])
map_proto_path = download_testdata(map_proto_url, map_proto, module="data")
label_path = download_testdata(label_map_url, label_map, module="data")
with tf_compat_v1.gfile.GFile(model_path, "rb") as f:
graph_def = tf_compat_v1.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name="")
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# Add shapes to the graph.
with tf_compat_v1.Session() as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, "softmax")
for img in pictures:
dataset.append(np.array(Image.open(img).resize((299, 299))))
shape_dict = {"data": [batch, 3, 299, 299]}
dtype_dict = {"DecodeJpeg/contents": "uint8"}
mod, params = relay.frontend.from_tensorflow(graph_def, layout=None, shape=shape_dict)
else:
raise Exception('Not Supported!')
# Build the graph
if device == 'x86':
target = "llvm"
ctx = tvm.cpu(0)
log_print(log, 'Target: ' + target)
elif device == 'Metal':
target = "metal"
ctx = tvm.metal(0)
log_print(log, 'Target: ' + target)
elif device == 'arm_cpu':
target = tvm.target.arm_cpu(remote["type"])
ctx = tvm.cpu(0)
log_print(log, 'Target: ' + remote["type"])
else:
target = device
ctx = tvm.cpu(0)
log_print(log, 'Target: ' + device)
log_print(log, 'Actual Model: ' + model + '\n')
print('Making the graph...')
if autotvm_log is not None:
from tvm import autotvm
log_print(log, 'Using AutoTVM file ' + autotvm_log)
with autotvm.apply_graph_best(autotvm_log):
with tvm.transform.PassContext(opt_level=3):
lib = relay.build(func, target, params=params)
else:
with tvm.transform.PassContext(opt_level=3):
lib = relay.build(func, target, params=params)
print("\nSetting up TVM...")
from tvm.contrib import graph_runtime
# Remote upload
if remote is not None:
from tvm import rpc
from tvm.contrib import utils, graph_runtime as runtime
print("Exporting graph...")
tmp = utils.tempdir()
lib_fname = tmp.relpath("net.tar")
lib.export_library(lib_fname)
print("Connecting to device...")
remote = rpc.connect(str(remote["ip"]), int(remote["port"]))
print("Uploading to device...")
remote.upload(lib_fname)
lib = remote.load_module("net.tar")
if device == 'x86':
ctx = remote.cpu(0)
elif device == 'Metal':
ctx = remote.metal(0)
elif device == 'arm_cpu':
ctx = remote.cpu(0)
else:
ctx = remote.cpu(0)
dtype = "float32"
m = graph_runtime.GraphModule(lib["default"](ctx))
def run_tvm(pics, number, repeat):
"""
Runs a single inference and gives back the time
:param pics: The images(s) to run
:param number: The number of times to run the inference
:param repeat: The number of times to repeat the measurement
:return: An array with the time and the result
"""
# combine pictures
arr = np.ndarray(shape=input_shape, dtype=dtype)
p = 0
for ip in pics:
arr[p] = ip.astype(dtype)
p = p + 1
m.set_input("data", tvm.nd.array(arr))
#Actually run inference
time = m.module.time_evaluator("run", ctx, number=number, repeat=repeat)()
#Get output
res = []
if platform == 'MXNet':
for i in range(len(pics)):
res.append(synset[np.argmax(m.get_output(0).asnumpy()[i])])
if platform == 'PyTorch':
# Get top-1 result for TVM
for i in range(len(pics)):
top1_tvm = np.argmax(m.get_output(0).asnumpy()[i])
tvm_class_key = class_id_to_key[top1_tvm]
res.append(key_to_classname[tvm_class_key])
if platform == 'TensorFlow':
pre = np.squeeze(m.get_output(0, tvm.nd.empty(((1, 1008)), "float32")).asnumpy())
node_lookup = tf_testing.NodeLookup(label_lookup_path=map_proto_path, uid_lookup_path=label_path)
top_k = pre.argsort()[-5:][::-1]
res = node_lookup.id_to_string(top_k[0])
return [time, res]
# Run the inferences
output = []
total = 0
print("\nRunning inferences...")
for i in range(int(len(dataset) / batch)):
log_print(log, "\nSet " + str(i + 1) + ":")
inp = []
# Create the next batch
for j in range(batch):
inp.append(dataset[batch * i + j])
# Run inference here
output = run_tvm(inp, runs, reps)
# Output results
e = 0
for rl in output[1]:
log_print(log, "Image " + str(e + 1) + " Path: " + pictures[batch * i + e])
log_print(log, "Image " + str(e + 1) + " ID: " + rl)
e = e + 1
log_print(log, "Time taken: " + str('%.2f' % (1000 * output[0].mean)) + " ms")
total = total + output[0].mean
ave = total / int(len(dataset) / batch)
log_print(log, '\nAVERAGE TIME: ' + str(ave * 1000) + " ms")
log_print(log, "Finished on " + str(datetime.now().strftime("%m/%d/%Y at %H:%M:%S")))
log.close()
return
model_name,
module=['tf', 'InceptionV1'])
map_proto_path = download_testdata(map_proto_url, map_proto, module='data')
label_path = download_testdata(label_map_url, label_map, module='data')
######################################################################
# Import model
# ------------
# Creates tensorflow graph definition from protobuf file.
with tf.gfile.FastGFile(model_path, 'rb') as f:
graph_def = tf.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# Add shapes to the graph.
with tf.Session() as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, 'softmax')
######################################################################
# Decode image
# ------------
# .. note::
#
# tensorflow frontend import doesn't support preprocessing ops like JpegDecode.
# JpegDecode is bypassed (just return source node).
# Hence we supply decoded frame to TVM instead.
#
from PIL import Image
def compile(info):
if info['model_path'].endswith('.onnx'):
is_onnx = True
elif info['model_path'].endswith('.pb'):
is_onnx = False
else:
raise Exception('Model file format not supported')
# Load model
if is_onnx:
onnx_model = onnx.load(info['model_path'])
mod, params = relay.frontend.from_onnx(onnx_model, info['input_dict'])
optimization_level = 3
else:
with tf.compat.v1.Session() as sess:
with tf.io.gfile.GFile(info['model_path'], 'rb') as f:
graph_def = tf.compat.v1.GraphDef()
graph_def.ParseFromString(f.read())
input_map = {}
for index, (name,
shape) in enumerate(info['input_dict'].items()):
tf_new_image = tf.compat.v1.placeholder(
shape=[1 if x == -1 else x for x in shape],
dtype=info['input_data_type'],
name=name)
input_map["input:" + str(index)] = tf_new_image
tf.import_graph_def(graph_def, name='', input_map=input_map)
graph_def = sess.graph.as_graph_def()
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
input_shape_dict = {'DecodeJpeg/contents': info['input_list']}
mod, params = relay.frontend.from_tensorflow(
graph_def, shape=input_shape_dict, outputs=info['output_names'])
optimization_level = 2
# Set compilation params
if info['cross_compile']:
if info['target'] == 'cuda':
raise Exception('cuda cross-compilation not supported yet')
info['target'] += ' -target=aarch64-linux-gnu'
# Transform data layout to what is expected by CUDA hardware, i.e. NCHW
if info['target'] == 'cuda':
desired_layouts = {'nn.conv2d': ['NCHW', 'default']}
seq = tvm.transform.Sequential([
relay.transform.RemoveUnusedFunctions(),
relay.transform.ConvertLayout(desired_layouts)
])
with tvm.transform.PassContext(opt_level=3):
mod = seq(mod)
# Compile model
# Note opt_level cannot be higher than 2 because of a bug:
# https://discuss.tvm.ai/t/tvm-0-6-1-compile-yolo-v2-tiny-fail-worked-in-v0-5-2/7244
with autotvm.apply_history_best(info['autotvm_log']):
with relay.build_config(opt_level=optimization_level):
graph, lib, params = relay.build(mod,
target=info['target'],
params=params)
# Write the compiled model to files
output_model_path = path.join(info['output_path'],
OUTPUT_NETWORK_MODULE_FILENAME)
output_graph_path = path.join(info['output_path'],
OUTPUT_NETWORK_GRAPH_FILENAME)
output_param_path = path.join(info['output_path'],
OUTPUT_NETWORK_PARAM_FILENAME)
print('Writing library to', output_model_path)
if info['cross_compile']:
lib.export_library(
output_model_path,
cc.build_create_shared_func(options=[
'--target=aarch64-linux-gnu', '-march=armv8-a', '-mfpu=NEON'
],
compile_cmd='/usr/bin/clang'))
else:
lib.export_library(output_model_path)
print('Writing graph to', output_graph_path)
with open(output_graph_path, 'w') as graph_file:
graph_file.write(graph)
print('Writing weights to', output_param_path)
with open(output_param_path, 'wb') as param_file:
param_file.write(relay.save_param_dict(params))
def test_forward_ptb():
'''test ptb model'''
config = tf_testing.get_config()
num_steps = config.num_steps
num_hidden = config.hidden_size
num_layers = config.num_layers
batch_size = config.batch_size
vocab_size = config.vocab_size
out_sample_shape = (batch_size, vocab_size)
out_state_shape = (num_layers, 2, batch_size, num_hidden)
#Sample input
inpt = "we have no useful information on"
cnt_sample = 20
def _pretty_print(items, is_char_model, id2word):
if not is_char_model:
return ' '.join([id2word[x] for x in items])
else:
return ''.join([id2word[x] for x in items]).replace('_', ' ')
def _get_tvm_graph_module(graph_def):
sym, params = nnvm.frontend.from_tensorflow(graph_def)
#Cell inputs 'c and 'h' consist of all layers values
shape_dict = {'Model/Placeholder': (batch_size, num_steps),
'Model/RNN/RNN/multi_rnn_cell/cell_0/lstm_cell/LSTMBlockCell_c':(num_layers, batch_size, num_hidden),
'Model/RNN/RNN/multi_rnn_cell/cell_0/lstm_cell/LSTMBlockCell_h':(num_layers, batch_size, num_hidden)}
dtype_dict = {'Model/Placeholder': 'int32',
'Model/RNN/RNN/multi_rnn_cell/cell_0/lstm_cell/LSTMBlockCell_c':'float32',
'Model/RNN/RNN/multi_rnn_cell/cell_0/lstm_cell/LSTMBlockCell_h':'float32'}
target = 'llvm'
graph, lib, params = nnvm.compiler.build(sym, target, shape_dict,
dtype=dtype_dict, params=params)
from tvm.contrib import graph_runtime
ctx = tvm.cpu(0)
return params, graph_runtime.create(graph, lib, ctx)
def _do_tvm_sample(model, data, in_states, params, num_samples):
"""Sampled from the model"""
samples = []
state = in_states
sample = None
def _get_sample(data, state):
input_data = np.full((batch_size, num_steps), data, dtype="int32")
in_state_tup = np.split(state, indices_or_sections=2, axis=1)
in_state_c = np.reshape(in_state_tup[0], (num_layers, batch_size, num_hidden))
in_state_h = np.reshape(in_state_tup[1], (num_layers, batch_size, num_hidden))
model.set_input('Model/Placeholder', tvm.nd.array(input_data.astype("int32")))
model.set_input('Model/RNN/RNN/multi_rnn_cell/cell_0/lstm_cell/LSTMBlockCell_c',
tvm.nd.array(in_state_c.astype("float32")))
model.set_input('Model/RNN/RNN/multi_rnn_cell/cell_0/lstm_cell/LSTMBlockCell_h',
tvm.nd.array(in_state_h.astype("float32")))
model.set_input(**params)
model.run()
tvm_output = model.get_output(0, tvm.nd.empty(out_sample_shape,
"float32")).asnumpy()
state_output = model.get_output(1, tvm.nd.empty(out_state_shape,
"float32")).asnumpy()
sample = tf_testing.pick_from_weight(tvm_output[0])
return sample, state_output
for x in data:
sample, state = _get_sample(x, state)
if sample is not None:
samples.append(sample)
else:
samples.append(0)
k = 1
while k < num_samples:
sample, state = _get_sample(samples[-1], state)
samples.append(sample)
k += 1
return samples, state
with tf.Graph().as_default():
word_to_id, id_to_word, graph_def = tf_testing.get_workload_ptb()
vocab_size = len(word_to_id)
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
sess = tf.Session()
#TVM graph module creation
params, m = _get_tvm_graph_module(graph_def)
# Create 10 predicted statments of 20 words
cnt_stm = 0
while cnt_stm < 10:
cnt_stm += 1
in_state = np.full((num_layers, 2, batch_size, num_hidden), 0, dtype="float32")
seed_for_sample = inpt.split()
tvm_samples, tvm_state = _do_tvm_sample(m, [word_to_id[word] \
for word in seed_for_sample],
in_state, params, cnt_sample)
tvm_sample_str = _pretty_print(tvm_samples, False, id_to_word)
tf_samples, tf_state = tf_testing.do_tf_sample(sess,
[word_to_id[word] for word in seed_for_sample],
in_state, cnt_sample)
tf_sample_str = _pretty_print(tf_samples, False, id_to_word)
inpt = tvm_sample_str
tvm.testing.assert_allclose(tf_samples, tvm_samples, rtol=1e-5, atol=1e-5)
assert(tvm_sample_str == tf_sample_str)
def test_TF():
model = TFNet()
tf.saved_model.save(obj=model, export_dir="./out/tensorflow/model")
# frozen_model
'''
TVM不支持直接读取tf模型，需要将模型转化成frozen_model
有点类似于动态模型转化成静态模型的感觉
代码是网上抄的，所以我不知道原理
'''
full_model = tf.function(lambda Input: model(Input))
full_model = full_model.get_concrete_function(
tf.TensorSpec((1, 3, 64, 64), 'float32'))
# Get frozen ConcreteFunction
frozen_func = convert_variables_to_constants_v2(full_model)
frozen_func.graph.as_graph_def()
layers = [op.name for op in frozen_func.graph.get_operations()]
# Save frozen graph from frozen ConcreteFunction to hard drive
tf.io.write_graph(graph_or_graph_def=frozen_func.graph,
logdir="./out/tensorflow/frozen_models",
name="test.pb",
as_text=False)
# graph_def
'''
根据frozen_model转化成可以被分析的graph_def形式，api来源于tensorflow1的GraphDef()
没有找到tensorflow2的支持信息，官方也没有给出信息
'''
with tf_compat_v1.gfile.GFile('./out/tensorflow/frozen_models/test.pb',
"rb") as f:
graph_def = tf_compat_v1.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name="")
# Call the utility to import the graph definition into default graph.
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
# with tf_compat_v1.Session() as sess:
# graph_def = tf_testing.AddShapesToGraphDef(sess, 'Identity')
# input_node
'''
输入节点标记
'''
input_node_name = [
node.name for node in graph_def.node
if len(node.input) == 0 and node.op not in ('Const')
]
input_node_name = input_node_name[0]
shape_dict = {input_node_name: (1, 3, 64, 64)}
# output_node
'''
寻找最终节点（无后继），默认是最后一个节点
'''
graph_dict = dict()
for node in graph_def.node:
graph_dict[node.name] = node.input
for name_src in graph_dict:
found = False
for name_dst in graph_dict:
if name_src == name_dst: continue
if name_src in graph_dict[name_dst]:
found = True
break
if not found:
output_node_name = name_src
break
# Addshape
'''
大概就是在output节点上打个标记，表示最终输出，默认情况下是最后一个节点
'''
with tf_compat_v1.Session(graph=graph) as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, output_node_name)
layout = 'NCHW'
mod, params = relay.frontend.from_tensorflow(graph_def,
layout=layout,
shape=shape_dict,
outputs=[output_node_name])
f = open('./out/tensorflow.txt', 'w')
print(mod['main'], file=f) ## 需要声明是main，现在的翻译转化会产生很多没有用的函数信息
f.close()
def compile(info):
if info['model_path'].endswith('.onnx'):
is_onnx = True
elif info['model_path'].endswith('.pb'):
is_onnx = False
else:
raise Exception('Model file format not supported')
# Load model
if is_onnx:
onnx_model = onnx.load(info['model_path'])
mod, params = relay.frontend.from_onnx(onnx_model, info['input_dict'])
optimization_level = 3
else:
with tf.compat.v1.Session() as sess:
with tf.io.gfile.GFile(info['model_path'], 'rb') as f:
graph_def = tf.compat.v1.GraphDef()
graph_def.ParseFromString(f.read())
tf.import_graph_def(graph_def, name='')
graph_def = sess.graph.as_graph_def()
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
input_shape_dict = {'DecodeJpeg/contents': info['input_list']}
mod, params = relay.frontend.from_tensorflow(
graph_def, shape=input_shape_dict, outputs=info['output_names'])
optimization_level = 2
# Set compilation params
target = 'llvm'
if info['cross_compile']:
target += ' -target=aarch64-linux-gnu'
# Compile model
# Note opt_level cannot be higher than 2 because of a bug:
# https://discuss.tvm.ai/t/tvm-0-6-1-compile-yolo-v2-tiny-fail-worked-in-v0-5-2/7244
with relay.build_config(opt_level=optimization_level):
graph, lib, params = relay.build(mod, target=target, params=params)
# Write the compiled model to files
output_model_path = path.join(info['output_path'],
OUTPUT_NETWORK_MODULE_FILENAME)
output_graph_path = path.join(info['output_path'],
OUTPUT_NETWORK_GRAPH_FILENAME)
output_param_path = path.join(info['output_path'],
OUTPUT_NETWORK_PARAM_FILENAME)
print('Writing library to', output_model_path)
if info['cross_compile']:
lib.export_library(
output_model_path,
cc.build_create_shared_func(options=[
'--target=aarch64-linux-gnu', '-march=armv8-a', '-mfpu=NEON'
],
compile_cmd='/usr/bin/clang'))
else:
lib.export_library(output_model_path)
print('Writing graph to', output_graph_path)
with open(output_graph_path, 'w') as graph_file:
graph_file.write(graph)
print('Writing weights to', output_param_path)
with open(output_param_path, 'wb') as param_file:
param_file.write(relay.save_param_dict(params))
def export_module(opts):
# Target settings
layout = "NCHW"
# Download required files
from tvm.contrib.download import download_testdata
model_path = download_testdata(model_url, model_file_name, module=['tf', 'keyword_spotting'])
label_path = download_testdata(label_url, label_name, module=['data'])
# Import model
with tf_compat_v1.gfile.GFile(model_path, 'rb') as f:
graph_def = tf_compat_v1.GraphDef()
graph_def.ParseFromString(f.read())
graph = tf.import_graph_def(graph_def, name='')
graph_def = tf_testing.ProcessGraphDefParam(graph_def)
with tf_compat_v1.Session() as sess:
graph_def = tf_testing.AddShapesToGraphDef(sess, 'labels_softmax')
build_dir = opts.out_dir
if not os.path.exists(build_dir):
os.makedirs(build_dir)
##save original TF graph
if DEBUG_LOG:
with open(os.path.join(build_dir, f'{model_name}_graph_original.log'), 'w') as orig_file:
orig_file.write(str(graph_def))
##remove pre-processing nodes and fix begining
nodes = []
##add first op
input_dim0 = 1
input_dim1 = 49
input_dim2 = 10
new_input = graph_def.node.add()
new_input.op = 'Placeholder'
new_input.name = 'Mfcc'
new_input.attr["dtype"].CopyFrom(attr_value_pb2.AttrValue(
type=dtypes.float32.as_datatype_enum))
nodes.append(new_input)
removed_count = 0
for ii, node in enumerate(graph_def.node, start=0):
if node.op == 'DecodeWav' \
or node.op == 'AudioSpectrogram' \
or node.op == 'Mfcc' \
or node.op == 'Placeholder' \
or node.op == 'wav_data':
removed_count += 1
pass
else:
nodes.append(node)
print(f'NUM of layers removed: {removed_count}')
new_graph = tf_compat_v1.GraphDef()
new_graph.node.extend(nodes)
##log new graph
if DEBUG_LOG:
with open(os.path.join(build_dir, f'{model_name}_graph_new.log'), 'w') as new_graph_log:
new_graph_log.write(str(new_graph))
##get mod and params with new graph
shape_dict = {'Mfcc': (1, 49, 10)}
mod, params = relay.frontend.from_tensorflow(new_graph,
layout=layout,
shape=shape_dict)
if DEBUG_LOG:
with open(os.path.join(build_dir, f'{model_name}_mod.log'), 'w') as mod_file:
mod_file.write(str(mod))
with open(os.path.join(build_dir, f'{model_name}_param.log'), 'w') as param_log:
param_log.write(str(params))
#quantization
if opts.quantize:
if not opts.global_scale:
raise RuntimeError('Global Scale is not valid!')
global_scale = float(opts.global_scale)
print('INFO: Quantizing...')
print(f'INFO: Global Scale: {global_scale}')
with relay.quantize.qconfig(calibrate_mode='global_scale',
global_scale=global_scale,
skip_conv_layers=[0]):
mod = relay.quantize.quantize(mod, params)
if DEBUG_LOG:
with open(os.path.join(build_dir, f'{model_name}_mod_quantized.log'), 'w') as mod_log:
mod_log.write(str(mod))
#save module
if opts.quantize:
file_path = f'{build_dir}/module_gs_{global_scale}.pickle'
with open(file_path, 'wb') as h1:
pickle.dump(mod, h1, protocol=pickle.HIGHEST_PROTOCOL)
print(f'INFO: {file_path} saved!')
with open(f'{build_dir}/module_gs_{global_scale}.txt', 'w') as f:
f.write(mod.astext())
else:
file_path = f'{build_dir}/module.pickle'
with open(file_path, 'wb') as h1:
pickle.dump(mod, h1, protocol=pickle.HIGHEST_PROTOCOL)
print(f'INFO: {file_path} saved!')
param_path = f'{build_dir}/params.bin'
with open(param_path, 'wb') as f_params:
f_params.write(relay.save_param_dict(params))
print(f'INFO: {param_path} saved!')
with open(f'{build_dir}/module.txt', 'w') as f:
f.write(mod.astext())
return mod, params