def create_models(models_dir, dtype, shape, no_batch=True):
model_version = 1
if FLAGS.graphdef:
create_tf_modelconfig(False, models_dir, model_version, 8, dtype, shape)
create_tf_modelfile(False, models_dir, model_version, 8, dtype, shape)
if no_batch:
create_tf_modelconfig(False, models_dir, model_version, 0, dtype, shape)
create_tf_modelfile(False, models_dir, model_version, 0, dtype, shape)
if FLAGS.savedmodel:
create_tf_modelconfig(True, models_dir, model_version, 8, dtype, shape)
create_tf_modelfile(True, models_dir, model_version, 8, dtype, shape)
if no_batch:
create_tf_modelconfig(True, models_dir, model_version, 0, dtype, shape)
create_tf_modelfile(True, models_dir, model_version, 0, dtype, shape)
if FLAGS.netdef:
create_netdef_modelconfig(models_dir, model_version, 8, dtype, shape)
create_netdef_modelfile(models_dir, model_version, 8, dtype, shape)
if no_batch:
create_netdef_modelconfig(models_dir, model_version, 0, dtype, shape)
create_netdef_modelfile(models_dir, model_version, 0, dtype, shape)
if FLAGS.tensorrt:
create_plan_modelconfig(models_dir, model_version, 8, dtype, shape + [1, 1])
create_plan_modelfile(models_dir, model_version, 8, dtype, shape + [1, 1])
if no_batch:
create_plan_modelconfig(models_dir, model_version, 0, dtype, shape + [1, 1])
create_plan_modelfile(models_dir, model_version, 0, dtype, shape + [1, 1])
if FLAGS.onnx:
create_onnx_modelconfig(models_dir, model_version, 8, dtype, shape)
create_onnx_modelfile(models_dir, model_version, 8, dtype, shape)
if no_batch:
create_onnx_modelconfig(models_dir, model_version, 0, dtype, shape)
create_onnx_modelfile(models_dir, model_version, 0, dtype, shape)
if FLAGS.ensemble:
for pair in emu.platform_types_and_validation():
if pair[0] == "plan":
shape = shape + [1, 1]
if not pair[1](dtype, dtype, dtype,
shape, shape, shape):
continue
emu.create_sequence_ensemble_modelconfig(
pair[0], models_dir, 8, model_version, shape, dtype)
emu.create_sequence_ensemble_modelfile(
pair[0], models_dir, 8, model_version, shape, dtype)
if no_batch:
emu.create_sequence_ensemble_modelconfig(
pair[0], models_dir, 0, model_version, shape, dtype)
emu.create_sequence_ensemble_modelfile(
pair[0], models_dir, 0, model_version, shape, dtype)
def create_models(
models_dir, input_dtype, output0_dtype, output1_dtype,
input_shape, output0_shape, output1_shape,
output0_label_cnt, version_policy=None):
model_version = 1
# Create two models, one that supports batching with a max-batch
# of 8, and one that does not with a max-batch of 0
if FLAGS.graphdef:
# max-batch 8
create_graphdef_modelconfig(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_graphdef_modelfile(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
# max-batch 0
create_graphdef_modelconfig(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_graphdef_modelfile(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
if FLAGS.savedmodel:
# max-batch 8
create_savedmodel_modelconfig(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_savedmodel_modelfile(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
# max-batch 0
create_savedmodel_modelconfig(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_savedmodel_modelfile(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
if FLAGS.netdef:
# max-batch 8
create_netdef_modelconfig(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_netdef_modelfile(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
# max-batch 0
create_netdef_modelconfig(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_netdef_modelfile(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
if FLAGS.tensorrt:
# max-batch 8
create_plan_modelconfig(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_plan_modelfile(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
# max-batch 0
create_plan_modelconfig(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_plan_modelfile(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
if FLAGS.onnx:
# max-batch 8
create_onnx_modelconfig(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_onnx_modelfile(
models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
# max-batch 0
create_onnx_modelconfig(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
create_onnx_modelfile(
models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
if FLAGS.ensemble:
for pair in emu.platform_types_and_validation():
if not pair[1](input_dtype, output0_dtype, output1_dtype,
input_shape, output0_shape, output1_shape):
continue
if pair[0] == "plan":
if len(input_shape) == 1:
input_shape = (input_shape[0], 1, 1)
if len(output0_shape) == 1:
output0_shape = (output0_shape[0], 1, 1)
if len(output1_shape) == 1:
output1_shape = (output1_shape[0], 1, 1)
# max-batch 8
emu.create_ensemble_modelconfig(
pair[0], models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
emu.create_ensemble_modelfile(
pair[0], models_dir, 8, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
# max-batch 0
emu.create_ensemble_modelconfig(
pair[0], models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy)
emu.create_ensemble_modelfile(
pair[0], models_dir, 0, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype)
create_plan_modelfile(FLAGS.models_dir, 0, 3,
(16,1,1), (16,1,1), (16,1,1), vt, vt, vt, swap=True)
if FLAGS.onnx:
for vt in [np.float16, np.float32, np.int8, np.int16, np.int32]:
create_onnx_modelfile(FLAGS.models_dir, 8, 2,
(16,), (16,), (16,), vt, vt, vt, swap=True)
create_onnx_modelfile(FLAGS.models_dir, 8, 3,
(16,), (16,), (16,), vt, vt, vt, swap=True)
create_onnx_modelfile(FLAGS.models_dir, 0, 2,
(16,), (16,), (16,), vt, vt, vt, swap=True)
create_onnx_modelfile(FLAGS.models_dir, 0, 3,
(16,), (16,), (16,), vt, vt, vt, swap=True)
if FLAGS.ensemble:
for pair in emu.platform_types_and_validation():
shape = (16, 1, 1) if pair[0] == "plan" else (16,)
for vt in [np.float16, np.float32, np.int8, np.int16, np.int32]:
if not pair[1](vt, vt, vt, shape, shape, shape):
continue
emu.create_ensemble_modelfile(pair[0], FLAGS.models_dir, 8, 2,
shape, shape, shape, vt, vt, vt, swap=True)
emu.create_ensemble_modelfile(pair[0], FLAGS.models_dir, 8, 3,
shape, shape, shape, vt, vt, vt, swap=True)
emu.create_ensemble_modelfile(pair[0], FLAGS.models_dir, 0, 2,
shape, shape, shape, vt, vt, vt, swap=True)
emu.create_ensemble_modelfile(pair[0], FLAGS.models_dir, 0, 3,
shape, shape, shape, vt, vt, vt, swap=True)
# Tests with models that accept variable-shape input/output tensors
if FLAGS.variable:
def create_models(models_dir, dtype, shape, no_batch=True):
model_version = 1
if FLAGS.graphdef:
create_tf_modelconfig(False, models_dir, model_version, 8, dtype,
shape)
create_tf_modelfile(False, models_dir, model_version, 8, dtype, shape)
if no_batch:
create_tf_modelconfig(False, models_dir, model_version, 0, dtype,
shape)
create_tf_modelfile(False, models_dir, model_version, 0, dtype,
shape)
if FLAGS.savedmodel:
create_tf_modelconfig(True, models_dir, model_version, 8, dtype, shape)
create_tf_modelfile(True, models_dir, model_version, 8, dtype, shape)
if no_batch:
create_tf_modelconfig(True, models_dir, model_version, 0, dtype,
shape)
create_tf_modelfile(True, models_dir, model_version, 0, dtype,
shape)
if FLAGS.tensorrt:
suffix = []
if dtype == np.int8:
suffix = [1, 1]
create_plan_modelconfig(models_dir, model_version, 8, dtype,
shape + suffix)
create_plan_modelfile(models_dir, model_version, 8, dtype,
shape + suffix)
if no_batch:
create_plan_modelconfig(models_dir, model_version, 0, dtype,
shape + suffix)
create_plan_modelfile(models_dir, model_version, 0, dtype,
shape + suffix)
if FLAGS.onnx:
create_onnx_modelconfig(models_dir, model_version, 8, dtype, shape)
create_onnx_modelfile(models_dir, model_version, 8, dtype, shape)
if no_batch:
create_onnx_modelconfig(models_dir, model_version, 0, dtype, shape)
create_onnx_modelfile(models_dir, model_version, 0, dtype, shape)
if FLAGS.libtorch:
create_libtorch_modelconfig(models_dir, model_version, 8, dtype, shape)
create_libtorch_modelfile(models_dir, model_version, 8, dtype, shape)
if no_batch:
create_libtorch_modelconfig(models_dir, model_version, 0, dtype,
shape)
create_libtorch_modelfile(models_dir, model_version, 0, dtype,
shape)
if FLAGS.ensemble:
for pair in emu.platform_types_and_validation():
config_shape = shape
if pair[0] == "plan" and dtype == np.int8:
config_shape = shape + [1, 1]
if not pair[1](dtype, dtype, dtype, config_shape, config_shape,
config_shape):
continue
emu.create_sequence_ensemble_modelconfig(pair[0], models_dir, 8,
model_version,
config_shape, dtype)
emu.create_sequence_ensemble_modelfile(pair[0], models_dir, 8,
model_version, config_shape,
dtype)
if no_batch:
emu.create_sequence_ensemble_modelconfig(
pair[0], models_dir, 0, model_version, config_shape, dtype)
emu.create_sequence_ensemble_modelfile(pair[0], models_dir, 0,
model_version,
config_shape, dtype)