def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True): model_version = 1 if FLAGS.graphdef: create_tf_modelconfig(False, models_dir, model_version, io_cnt, 8, dtype, shape) create_tf_modelfile(False, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_tf_modelconfig(False, models_dir, model_version, io_cnt, 0, dtype, shape) create_tf_modelfile(False, models_dir, model_version, io_cnt, 0, dtype, shape) if FLAGS.savedmodel: create_tf_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_tf_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_tf_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_tf_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape) if FLAGS.netdef: create_netdef_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_netdef_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_netdef_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_netdef_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape) if FLAGS.ensemble: emu.create_nop_modelconfig(models_dir, shape, dtype) create_ensemble_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_ensemble_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_ensemble_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_ensemble_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape)
if not FLAGS.variable: create_models(FLAGS.models_dir, np.float32, [ 1, ]) create_models(FLAGS.models_dir, np.int32, [ 1, ]) create_models(FLAGS.models_dir, np_dtype_string, [ 1, ]) # Tests with models that accept variable-shape input/output tensors if FLAGS.variable: create_models(FLAGS.models_dir, np.int32, [ -1, ], False) create_models(FLAGS.models_dir, np.float32, [ -1, ], False) create_models(FLAGS.models_dir, np_dtype_string, [ -1, ], False) if FLAGS.ensemble: # Create nop models used in ensemble for model_dtype in ["TYPE_INT32", "TYPE_FP32"]: # 3D shape for TensorRT Plan for model_shape in [(-1, ), (-1, -1, -1)]: emu.create_nop_modelconfig(FLAGS.models_dir, model_shape, model_dtype)
def create_models(models_dir, dtype, input_shapes, input_model_shapes, output_shapes=None, output_model_shapes=None, no_batch=True): model_version = 1 if output_shapes is None: output_shapes = input_shapes if output_model_shapes is None: output_model_shapes = input_model_shapes if FLAGS.graphdef: create_tf_modelconfig(False, models_dir, model_version, 8, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_tf_modelfile(False, models_dir, model_version, 8, dtype, input_model_shapes, output_model_shapes) if no_batch: create_tf_modelconfig(False, models_dir, model_version, 0, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_tf_modelfile(False, models_dir, model_version, 0, dtype, input_model_shapes, output_model_shapes) if FLAGS.savedmodel: create_tf_modelconfig(True, models_dir, model_version, 8, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_tf_modelfile(True, models_dir, model_version, 8, dtype, input_model_shapes, output_model_shapes) if no_batch: create_tf_modelconfig(True, models_dir, model_version, 0, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_tf_modelfile(True, models_dir, model_version, 0, dtype, input_model_shapes, output_model_shapes) if FLAGS.netdef: create_netdef_modelconfig(models_dir, model_version, 8, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_netdef_modelfile(models_dir, model_version, 8, dtype, input_model_shapes, output_model_shapes) if no_batch: create_netdef_modelconfig(models_dir, model_version, 0, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_netdef_modelfile(models_dir, model_version, 0, dtype, input_model_shapes, output_model_shapes) if FLAGS.onnx: create_onnx_modelconfig(models_dir, model_version, 8, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_onnx_modelfile(models_dir, model_version, 8, dtype, input_model_shapes, output_model_shapes) if no_batch: create_onnx_modelconfig(models_dir, model_version, 0, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_onnx_modelfile(models_dir, model_version, 0, dtype, input_model_shapes, output_model_shapes) # Shouldn't create ensembles that reshape to zero-sized tensors. Reshaping # from / to zero dimension is not allow as ensemble inputs / outputs # are passed from / to other model AS IF direct inference from client. # But create it anyway, expecting that the ensemble models can be served but # they will always return error message. if FLAGS.ensemble: # Create fixed size nop for ensemble models for shape in input_model_shapes: emu.create_nop_modelconfig(models_dir, shape, np.float32) emu.create_nop_tunnel_modelconfig(models_dir, shape, np.float32) emu.create_nop_modelconfig(models_dir, [-1], np.float32) create_ensemble_modelconfig(models_dir, model_version, 8, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_ensemble_modelfile(models_dir, model_version, 8, dtype, input_model_shapes, output_model_shapes) if no_batch: create_ensemble_modelconfig(models_dir, model_version, 0, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes) create_ensemble_modelfile(models_dir, model_version, 0, dtype, input_model_shapes, output_model_shapes)
def create_models(models_dir, dtype, shape, io_cnt=1, no_batch=True): model_version = 1 if FLAGS.graphdef: create_tf_modelconfig(False, models_dir, model_version, io_cnt, 8, dtype, shape) create_tf_modelfile(False, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_tf_modelconfig(False, models_dir, model_version, io_cnt, 0, dtype, shape) create_tf_modelfile(False, models_dir, model_version, io_cnt, 0, dtype, shape) if FLAGS.savedmodel: create_tf_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_tf_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_tf_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_tf_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape) if FLAGS.netdef: create_netdef_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_netdef_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_netdef_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_netdef_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape) if FLAGS.onnx: create_onnx_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_onnx_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_onnx_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_onnx_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape) if FLAGS.libtorch: create_libtorch_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_libtorch_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_libtorch_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_libtorch_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape) if FLAGS.tensorrt: create_plan_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_plan_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape, 32) if no_batch: create_plan_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_plan_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape, 32) if FLAGS.tensorrt_big: create_plan_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_plan_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape, 16 * 1024 * 1024) if no_batch: create_plan_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_plan_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape, 16 * 1024 * 1024) if FLAGS.ensemble: emu.create_nop_modelconfig(models_dir, shape, dtype) create_ensemble_modelconfig(True, models_dir, model_version, io_cnt, 8, dtype, shape) create_ensemble_modelfile(True, models_dir, model_version, io_cnt, 8, dtype, shape) if no_batch: create_ensemble_modelconfig(True, models_dir, model_version, io_cnt, 0, dtype, shape) create_ensemble_modelfile(True, models_dir, model_version, io_cnt, 0, dtype, shape)