class runner_wrapper:
def __init__(self, batch_size=1):
self.batch_size = batch_size
self.data = np.random.rand(batch_size, *shape).astype(np.float32)
def __call__(self, data_size):
with options({
'layout_optimizer': False,
'function_optimization': False
}):
for _ in range(data_size // self.batch_size):
ret = grap_model(self.data)
runner = runner_wrapper(batch_size=batch_size)
return runner
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="benchmark of tf/xla")
parser.add_argument("model", help="tf model name")
parser.add_argument("--batch", type=int, default=1, help='batch size')
parser.add_argument("--size", type=int, default=256, help='data size')
args = parser.parse_args()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
runner = grappler_runner(args.model, batch_size=args.batch)
duration = util.simple_bench(runner, data_size=args.size)
print(duration)
signature_keys = list(saved_model_loaded.signatures.keys())
print(signature_keys)
infer = saved_model_loaded.signatures['serving_default']
data = np.random.rand(batch_size, *shape).astype(np.float32)
x = tf.constant(data)
def runner(data_size):
for _ in range(data_size // batch_size):
infer(x)
return runner
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="benchmark of tf/TensorRT")
parser.add_argument("model", help="tf model name")
parser.add_argument("--batch", type=int, default=1, help='batch size')
parser.add_argument("--size", type=int, default=256, help='test data size')
args = parser.parse_args()
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
runner = tf2trt_runner(args.model, batch_size=args.batch)
throughput = util.simple_bench(runner,
data_size=args.size,
warmup=1,
rounds=5,
verbose=True)
print(throughput)
dtype = "float32"
data = tvm.nd.array(data.astype(dtype))
def runner(data_size):
for _ in range(data_size // batch_size):
module.set_input(input_name, data)
module.run()
return runner
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="benchmark of onnx/tvm")
parser.add_argument("model", help="onnx model name")
parser.add_argument("--backend",
choices=['cuda', 'llvm'],
default='cuda',
help='backend target to run')
parser.add_argument("--batch", type=int, default=1, help='batch size')
parser.add_argument("--size", type=int, default=256, help='data size')
args = parser.parse_args()
os.environ['TVM_BACKTRACE'] = '1'
runner = onnx2tvm_runner(args.model,
batch_size=args.batch,
backend=args.backend)
duration = util.simple_bench(runner, args.size)
print(duration)
x = np.random.rand(args.batch, *shape).astype(np.float32)
model.predict(x, callbacks=[tensorboard_callback])
if args.profile != "":
# from tensorflow_core.python.eager import profiler
log_dir = os.path.join(log_dir, args.profile)
# profiler = tf.profiler.experimental.Profile(log_dir)
runner = xla(args.model,
batch_size=args.batch,
xla=args.xla,
device=args.device)
util.simple_bench(runner,
data_size=args.size,
warmup=args.warmup,
rounds=args.rounds,
verbose=True)
tf.profiler.experimental.start(log_dir)
throughput = util.simple_bench(runner,
data_size=args.size,
warmup=args.warmup,
rounds=args.rounds,
verbose=True)
tf.profiler.experimental.stop()
else:
runner = xla(args.model,
batch_size=args.batch,
xla=args.xla,
device=args.device)