def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True):
model_version = 1
if FLAGS.graphdef:
create_tf_modelconfig(False, models_dir, model_version, io_cnt, 8, dtype, shape)
create_tf_modelfile(False, models_dir, model_version, io_cnt, 8, dtype, shape)
if no_batch:
create_tf_modelconfig(False, models_dir, model_version, io_cnt, 0, dtype, shape)
create_tf_modelfile(False, models_dir, model_version, io_cnt, 0, dtype, shape)
if FLAGS.savedmodel:
create_tf_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape)
create_tf_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape)
if no_batch:
create_tf_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape)
create_tf_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape)
if FLAGS.netdef:
create_netdef_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape)
create_netdef_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape)
if no_batch:
create_netdef_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape)
create_netdef_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape)
if FLAGS.ensemble:
emu.create_nop_modelconfig(models_dir, shape, dtype)
create_ensemble_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape)
create_ensemble_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape)
if no_batch:
create_ensemble_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape)
create_ensemble_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape)
if not FLAGS.variable:
create_models(FLAGS.models_dir, np.float32, [
1,
])
create_models(FLAGS.models_dir, np.int32, [
1,
])
create_models(FLAGS.models_dir, np_dtype_string, [
1,
])
# Tests with models that accept variable-shape input/output tensors
if FLAGS.variable:
create_models(FLAGS.models_dir, np.int32, [
-1,
], False)
create_models(FLAGS.models_dir, np.float32, [
-1,
], False)
create_models(FLAGS.models_dir, np_dtype_string, [
-1,
], False)
if FLAGS.ensemble:
# Create nop models used in ensemble
for model_dtype in ["TYPE_INT32", "TYPE_FP32"]:
# 3D shape for TensorRT Plan
for model_shape in [(-1, ), (-1, -1, -1)]:
emu.create_nop_modelconfig(FLAGS.models_dir, model_shape,
model_dtype)
def create_models(models_dir,
dtype,
input_shapes,
input_model_shapes,
output_shapes=None,
output_model_shapes=None,
no_batch=True):
model_version = 1
if output_shapes is None:
output_shapes = input_shapes
if output_model_shapes is None:
output_model_shapes = input_model_shapes
if FLAGS.graphdef:
create_tf_modelconfig(False, models_dir, model_version, 8, dtype,
input_shapes, input_model_shapes, output_shapes,
output_model_shapes)
create_tf_modelfile(False, models_dir, model_version, 8, dtype,
input_model_shapes, output_model_shapes)
if no_batch:
create_tf_modelconfig(False, models_dir, model_version, 0, dtype,
input_shapes, input_model_shapes,
output_shapes, output_model_shapes)
create_tf_modelfile(False, models_dir, model_version, 0, dtype,
input_model_shapes, output_model_shapes)
if FLAGS.savedmodel:
create_tf_modelconfig(True, models_dir, model_version, 8, dtype,
input_shapes, input_model_shapes, output_shapes,
output_model_shapes)
create_tf_modelfile(True, models_dir, model_version, 8, dtype,
input_model_shapes, output_model_shapes)
if no_batch:
create_tf_modelconfig(True, models_dir, model_version, 0, dtype,
input_shapes, input_model_shapes,
output_shapes, output_model_shapes)
create_tf_modelfile(True, models_dir, model_version, 0, dtype,
input_model_shapes, output_model_shapes)
if FLAGS.netdef:
create_netdef_modelconfig(models_dir, model_version, 8, dtype,
input_shapes, input_model_shapes,
output_shapes, output_model_shapes)
create_netdef_modelfile(models_dir, model_version, 8, dtype,
input_model_shapes, output_model_shapes)
if no_batch:
create_netdef_modelconfig(models_dir, model_version, 0, dtype,
input_shapes, input_model_shapes,
output_shapes, output_model_shapes)
create_netdef_modelfile(models_dir, model_version, 0, dtype,
input_model_shapes, output_model_shapes)
if FLAGS.onnx:
create_onnx_modelconfig(models_dir, model_version, 8, dtype,
input_shapes, input_model_shapes, output_shapes,
output_model_shapes)
create_onnx_modelfile(models_dir, model_version, 8, dtype,
input_model_shapes, output_model_shapes)
if no_batch:
create_onnx_modelconfig(models_dir, model_version, 0, dtype,
input_shapes, input_model_shapes,
output_shapes, output_model_shapes)
create_onnx_modelfile(models_dir, model_version, 0, dtype,
input_model_shapes, output_model_shapes)
# Shouldn't create ensembles that reshape to zero-sized tensors. Reshaping
# from / to zero dimension is not allow as ensemble inputs / outputs
# are passed from / to other model AS IF direct inference from client.
# But create it anyway, expecting that the ensemble models can be served but
# they will always return error message.
if FLAGS.ensemble:
# Create fixed size nop for ensemble models
for shape in input_model_shapes:
emu.create_nop_modelconfig(models_dir, shape, np.float32)
emu.create_nop_tunnel_modelconfig(models_dir, shape, np.float32)
emu.create_nop_modelconfig(models_dir, [-1], np.float32)
create_ensemble_modelconfig(models_dir, model_version, 8, dtype,
input_shapes, input_model_shapes,
output_shapes, output_model_shapes)
create_ensemble_modelfile(models_dir, model_version, 8, dtype,
input_model_shapes, output_model_shapes)
if no_batch:
create_ensemble_modelconfig(models_dir, model_version, 0, dtype,
input_shapes, input_model_shapes,
output_shapes, output_model_shapes)
create_ensemble_modelfile(models_dir, model_version, 0, dtype,
input_model_shapes, output_model_shapes)
def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True):
model_version = 1
if FLAGS.graphdef:
create_tf_modelconfig(False, models_dir, model_version, io_cnt, 8,
dtype, shape)
create_tf_modelfile(False, models_dir, model_version, io_cnt, 8, dtype,
shape)
if no_batch:
create_tf_modelconfig(False, models_dir, model_version, io_cnt, 0,
dtype, shape)
create_tf_modelfile(False, models_dir, model_version, io_cnt, 0,
dtype, shape)
if FLAGS.savedmodel:
create_tf_modelconfig(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
create_tf_modelfile(True, models_dir, model_version, io_cnt, 8, dtype,
shape)
if no_batch:
create_tf_modelconfig(True, models_dir, model_version, io_cnt, 0,
dtype, shape)
create_tf_modelfile(True, models_dir, model_version, io_cnt, 0,
dtype, shape)
if FLAGS.netdef:
create_netdef_modelconfig(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
create_netdef_modelfile(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
if no_batch:
create_netdef_modelconfig(True, models_dir, model_version, io_cnt,
0, dtype, shape)
create_netdef_modelfile(True, models_dir, model_version, io_cnt, 0,
dtype, shape)
if FLAGS.onnx:
create_onnx_modelconfig(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
create_onnx_modelfile(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
if no_batch:
create_onnx_modelconfig(True, models_dir, model_version, io_cnt, 0,
dtype, shape)
create_onnx_modelfile(True, models_dir, model_version, io_cnt, 0,
dtype, shape)
if FLAGS.libtorch:
create_libtorch_modelconfig(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
create_libtorch_modelfile(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
if no_batch:
create_libtorch_modelconfig(True, models_dir, model_version,
io_cnt, 0, dtype, shape)
create_libtorch_modelfile(True, models_dir, model_version, io_cnt,
0, dtype, shape)
if FLAGS.tensorrt:
create_plan_modelconfig(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
create_plan_modelfile(True, models_dir, model_version, io_cnt, 8,
dtype, shape, 32)
if no_batch:
create_plan_modelconfig(True, models_dir, model_version, io_cnt, 0,
dtype, shape)
create_plan_modelfile(True, models_dir, model_version, io_cnt, 0,
dtype, shape, 32)
if FLAGS.tensorrt_big:
create_plan_modelconfig(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
create_plan_modelfile(True, models_dir, model_version, io_cnt, 8,
dtype, shape, 16 * 1024 * 1024)
if no_batch:
create_plan_modelconfig(True, models_dir, model_version, io_cnt, 0,
dtype, shape)
create_plan_modelfile(True, models_dir, model_version, io_cnt, 0,
dtype, shape, 16 * 1024 * 1024)
if FLAGS.ensemble:
emu.create_nop_modelconfig(models_dir, shape, dtype)
create_ensemble_modelconfig(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
create_ensemble_modelfile(True, models_dir, model_version, io_cnt, 8,
dtype, shape)
if no_batch:
create_ensemble_modelconfig(True, models_dir, model_version,
io_cnt, 0, dtype, shape)
create_ensemble_modelfile(True, models_dir, model_version, io_cnt,
0, dtype, shape)
