def get_backend(backend):
if backend == "tensorflow":
from backend_tf import BackendTensorflow
backend = BackendTensorflow()
elif backend == "onnxruntime":
from backend_onnxruntime import BackendOnnxruntime
backend = BackendOnnxruntime()
elif backend == "null":
from backend_null import BackendNull
backend = BackendNull()
elif backend == "pytorch":
from backend_pytorch import BackendPytorch
backend = BackendPytorch()
elif backend == "pytorch-native":
from backend_pytorch_native import BackendPytorchNative
backend = BackendPytorchNative()
elif backend == "tflite":
from backend_tflite import BackendTflite
backend = BackendTflite()
elif backend == "tflite-calibrate":
from backend_tflite_calibrate import BackendTflite
backend = BackendTflite()
elif backend == "tflite-ncore":
from backend_tflite_ncore import BackendTfliteNcore
backend = BackendTfliteNcore()
elif backend == "tflite-ncore-offline-imagenet":
from backend_tflite_ncore_offline_imagenet import BackendTfliteNcoreOfflineImagenet
backend = BackendTfliteNcoreOfflineImagenet()
elif backend == "tflite-ncore-offline-ssd":
from backend_tflite_ncore_offline_ssd import BackendTfliteNcoreOfflineSSD
backend = BackendTfliteNcoreOfflineSSD()
else:
raise ValueError("unknown backend: " + backend)
return backend
def get_backend(backend):
if backend == "tensorflow":
from backend_tf import BackendTensorflow
backend = BackendTensorflow()
elif backend == "onnxruntime":
from backend_onnxruntime import BackendOnnxruntime
backend = BackendOnnxruntime()
elif backend == "null":
from backend_null import BackendNull
backend = BackendNull()
elif backend == "pytorch":
from backend_pytorch import BackendPytorch
backend = BackendPytorch()
elif backend == "pytorch-native":
from backend_pytorch_native import BackendPytorchNative
backend = BackendPytorchNative()
elif backend == "tflite":
from backend_tflite import BackendTflite
backend = BackendTflite()
elif backend == "tvm":
from backend_tvm import BackendTvm
backend = BackendTvm()
else:
raise ValueError("unknown backend: " + backend)
return backend
def get_backend(backend):
if backend == "tensorflow":
from backend_tf import BackendTensorflow
backend = BackendTensorflow()
elif backend == "null":
from backend_null import BackendNull
backend = BackendNull()
elif backend == "openvino":
from backend_openvino import BackendOpenvino
backend = BackendOpenvino()
else:
raise ValueError("unknown backend: " + backend)
return backend
def create_backend_instance(backend_type, args):
if backend_type == "tensorflow":
from backend_tf import BackendTensorflow
backend = BackendTensorflow(model_path,
batchsize,
inputs=None,
outputs=None)
backend.load(model_path, inputs=args.inputs, outputs=args.outputs)
elif backend_type == "onnxruntime":
from backend.onnx_backend import BackendOnnxruntime
backend = BackendOnnxruntime(args.batchsize)
elif backend_type == "null":
from backend_null import BackendNull
backend = BackendNull()
elif backend_type == "acl":
from backend.acl_backend import BackendAcl
backend = BackendAcl(args.batchsize)
backend.load(args.model,
inputs=args.inputs,
outputs=args.outputs,
device_id=args.device_id)
else:
raise ValueError("unknown backend: ", backend_type)
return backend
def get_backend(backend, dataset_path, dataset_calibration_list):
if backend == "tensorflow":
from backend_tf import BackendTensorflow
backend = BackendTensorflow()
elif backend == "onnxruntime":
from backend_onnxruntime import BackendOnnxruntime
backend = BackendOnnxruntime()
elif backend == "null":
from backend_null import BackendNull
backend = BackendNull()
elif backend == "pytorch":
from backend_pytorch import BackendPytorch
backend = BackendPytorch()
elif backend == "pytorch-native":
from backend_pytorch_native import BackendPytorchNative
backend = BackendPytorchNative()
elif backend == "pytorch-jit-traced":
from backend_pytorch_jit_traced import BackendPytorchJITTraced
backend = BackendPytorchJITTraced()
elif backend == "pytorch-fp32":
from backend_pytorch_fp32 import BackendPytorchFP32
backend = BackendPytorchFP32()
elif backend == "pytorch-ssd-jit-traced":
from backend_pytorch_ssd_jit_traced import BackendPytorchSSDJITTraced
backend = BackendPytorchSSDJITTraced()
elif backend == "pytorch-yolov3-jit-traced":
from backend_pytorch_yolov3_jit_traced import BackendPytorchYOLOv3JITTraced
backend = BackendPytorchYOLOv3JITTraced()
elif backend == "pytorch-yolov3-fp32":
from backend_pytorch_yolov3_fp32 import BackendPytorchYOLOv3FP32
backend = BackendPytorchYOLOv3FP32()
elif backend == "tflite":
from backend_tflite import BackendTflite
backend = BackendTflite()
elif backend == "edgecortix":
from backend_edgecortix import BackendEdgecortix
backend = BackendEdgecortix(dataset_path, dataset_calibration_list)
else:
raise ValueError("unknown backend: " + backend)
return backend
from backend_pytorch import BackendPytorch
backend = BackendPytorch()
elif backend == "pytorch-native":
from backend_pytorch_native import BackendPytorchNative
backend = BackendPytorchNative()
elif backend == "tflite":
from backend_tflite import BackendTflite
backend = BackendTflite()
else:
raise ValueError("unknown backend: " + backend)
return backend
#
# def main_model():
# main_model.args = get_args()
backend = BackendTensorflow()
def load_model():
global backend
# find backend
backend = get_backend("tensorflow")
# load model to backend
t1 = time.time()
model = backend.load("/model/resnet34_tf.22.1.pb", inputs=['image:0'], outputs=['detection_bboxes:0','detection_classes:0','detection_scores:0'])
t2 = time.time()
print("load model time= %f" % (t2 - t1))
#
# make one pass over the dataset to validate accuracy
def get_backend(backend):
if backend == "tensorflow":
from backend_tf import BackendTensorflow
backend = BackendTensorflow()
elif backend == "onnxruntime":
from backend_onnxruntime import BackendOnnxruntime
backend = BackendOnnxruntime()
elif backend == "null":
from backend_null import BackendNull
backend = BackendNull()
elif backend == "pytorch":
from backend_pytorch import BackendPytorch
backend = BackendPytorch()
elif backend == "pytorch-native":
from backend_pytorch_native import BackendPytorchNative
backend = BackendPytorchNative()
elif backend == "pytorch-centaur":
from backend_pytorch_centaur import BackendPytorchCentaur
backend = BackendPytorchCentaur()
elif backend == "pytorch-native-calibrate":
from backend_pytorch_native_calibrate import BackendPytorchNativeCalibrate
backend = BackendPytorchNativeCalibrate()
elif backend == "tflite":
from backend_tflite import BackendTflite
backend = BackendTflite()
elif backend == "tflite-calibrate":
from backend_tflite_calibrate import BackendTflite
backend = BackendTflite()
elif backend == "tflite-ncore":
from backend_tflite_ncore import BackendTfliteNcore
backend = BackendTfliteNcore()
elif backend == "tflite-ncore-mobilenet":
from backend_libncoretflite import BackendTfliteNcoreMobileNetV1
backend = BackendTfliteNcoreMobileNetV1()
backend.inputs = ["image_tensor:0"]
elif backend == "tflite-ncore-resnet":
from backend_libncoretflite import BackendTfliteNcoreResnet
backend = BackendTfliteNcoreResnet()
backend.inputs = ["image_tensor:0"]
elif backend == "tflite-ncore-ssd":
from backend_libncoretflite import BackendTfliteNcoreSSD
backend = BackendTfliteNcoreSSD()
backend.inputs = ["image_tensor:0"]
elif backend == "tflite-ncore-mobilenet-offline":
from backend_libncoretflite import BackendTfliteNcoreMobileNetV1Offline
backend = BackendTfliteNcoreMobileNetV1Offline()
backend.inputs = ["image_tensor:0"]
elif backend == "tflite-ncore-resnet-offline":
from backend_libncoretflite import BackendTfliteNcoreResnetOffline
backend = BackendTfliteNcoreResnetOffline()
backend.inputs = ["image_tensor:0"]
elif backend == "tflite-ncore-ssd-offline":
from backend_libncoretflite import BackendTfliteNcoreSSDOffline
backend = BackendTfliteNcoreSSDOffline()
backend.inputs = ["image_tensor:0"]
else:
raise ValueError("unknown backend: " + backend)
return backend
def main(argv):
del argv
global last_timeing
if FLAGS.scenario == "Server":
# Disable garbage collection for realtime performance.
gc.disable()
# define backend
backend = BackendTensorflow()
# override image format if given
image_format = FLAGS.data_format if FLAGS.data_format else backend.image_format(
)
# dataset to use
wanted_dataset, pre_proc, post_proc, kwargs = SUPPORTED_DATASETS[
FLAGS.dataset]
ds = wanted_dataset(data_path=FLAGS.dataset_path,
image_list=FLAGS.dataset_list,
name=FLAGS.dataset,
image_format=image_format,
use_cache=FLAGS.cache,
count=FLAGS.count,
cache_dir=FLAGS.cache_dir,
annotation_file=FLAGS.annotation_file,
use_space_to_depth=FLAGS.use_space_to_depth)
# load model to backend
# TODO(wangtao): parse flags to params.
params = dict(ssd_model.default_hparams().values())
params["conv0_space_to_depth"] = FLAGS.use_space_to_depth
params["use_bfloat16"] = FLAGS.use_bfloat16
params["use_fused_bn"] = FLAGS.use_fused_bn
masters = []
tpu_names = FLAGS.tpu_name
tpu_names = tpu_names.split(",")
for tpu_name in tpu_names:
tpu_cluster_resolver = tf.contrib.cluster_resolver.TPUClusterResolver(
tpu_name, zone=FLAGS.tpu_zone, project=FLAGS.gcp_project)
masters.append(tpu_cluster_resolver.get_master())
#
# make one pass over the dataset to validate accuracy
#
count = FLAGS.count if FLAGS.count else ds.get_item_count()
#
# warmup
#
log.info("warmup ...")
batch_size = FLAGS.batch_size[0] if FLAGS.scenario == "Offline" else 1
backend_lists = []
for _ in range(len(tpu_names)):
backend = BackendTensorflow()
backend_lists.append(backend)
runner = QueueRunner(backend_lists,
ds,
FLAGS.threads,
post_proc=post_proc,
max_batchsize=batch_size)
runner.start_run({}, FLAGS.accuracy)
def issue_queries(query_samples):
for i in [1]:
runner.enqueue(query_samples)
def flush_queries():
pass
def process_latencies(latencies_ns):
# called by loadgen to show us the recorded latencies
global last_timeing
last_timeing = [t / NANO_SEC for t in latencies_ns]
tf.logging.info("starting {}, latency={}".format(FLAGS.scenario,
FLAGS.max_latency))
settings = lg.TestSettings()
tf.logging.info(FLAGS.scenario)
settings.scenario = SCENARIO_MAP[FLAGS.scenario]
settings.qsl_rng_seed = FLAGS.qsl_rng_seed
settings.sample_index_rng_seed = FLAGS.sample_index_rng_seed
settings.schedule_rng_seed = FLAGS.schedule_rng_seed
if FLAGS.accuracy:
settings.mode = lg.TestMode.AccuracyOnly
else:
settings.mode = lg.TestMode.PerformanceOnly
if FLAGS.qps:
qps = float(FLAGS.qps)
settings.server_target_qps = qps
settings.offline_expected_qps = qps
if FLAGS.time:
settings.min_duration_ms = FLAGS.time * MILLI_SEC
settings.max_duration_ms = 0
qps = FLAGS.qps or 100
settings.min_query_count = qps * FLAGS.time
settings.max_query_count = 0
else:
settings.min_query_count = 270336
settings.max_query_count = 0
target_latency_ns = int(float(FLAGS.max_latency) * NANO_SEC)
settings.single_stream_expected_latency_ns = target_latency_ns
settings.multi_stream_target_latency_ns = target_latency_ns
settings.server_target_latency_ns = target_latency_ns
log_settings = lg.LogSettings()
log_settings.log_output.outdir = tempfile.mkdtemp()
log_settings.log_output.copy_detail_to_stdout = True
log_settings.log_output.copy_summary_to_stdout = True
log_settings.enable_trace = False
def load_query_samples(sample_list):
"""Load query samples and warmup the model."""
ds.load_query_samples(sample_list)
data = ds.get_image_list_inmemory()
def init_fn(cloud_tpu_id):
tf.logging.info("Load model for %dth cloud tpu", cloud_tpu_id)
runner.models[cloud_tpu_id].load(
FLAGS.model,
FLAGS.output_model_dir,
data,
params,
batch_size=FLAGS.batch_size,
master=masters[cloud_tpu_id],
scenario=FLAGS.scenario,
batch_timeout_micros=FLAGS.batch_timeout_micros)
# Init TPU.
for it in range(FLAGS.init_iterations):
tf.logging.info("Initialize cloud tpu at iteration %d", it)
for batch_size in FLAGS.batch_size:
example, _ = ds.get_indices([sample_list[0]] * batch_size)
_ = runner.models[cloud_tpu_id].predict(example)
threads = []
for i in range(len(tpu_names)):
thread = threading.Thread(target=init_fn, args=(i, ))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
sut = lg.ConstructSUT(issue_queries, flush_queries, process_latencies)
qsl = lg.ConstructQSL(count, min(count, 350), load_query_samples,
ds.unload_query_samples)
lg.StartTestWithLogSettings(sut, qsl, settings, log_settings)
runner.finish()
lg.DestroyQSL(qsl)
lg.DestroySUT(sut)
tf.io.gfile.mkdir(FLAGS.outdir)
for oldfile in tf.gfile.Glob(
os.path.join(log_settings.log_output.outdir, "*")):
basename = os.path.basename(oldfile)
newfile = os.path.join(FLAGS.outdir, basename)
tf.gfile.Copy(oldfile, newfile, overwrite=True)
if FLAGS.accuracy:
with tf.gfile.Open(os.path.join(FLAGS.outdir, "results.txt"),
"w") as f:
results = {"mAP": accuracy_coco.main()}
json.dump(results, f, sort_keys=True, indent=4)
