def create_models(models_dir, dtype, shape, no_batch=True): model_version = 1 if FLAGS.graphdef: create_tf_modelconfig(False, models_dir, model_version, 8, dtype, shape) create_tf_modelfile(False, models_dir, model_version, 8, dtype, shape) if no_batch: create_tf_modelconfig(False, models_dir, model_version, 0, dtype, shape) create_tf_modelfile(False, models_dir, model_version, 0, dtype, shape) if FLAGS.savedmodel: create_tf_modelconfig(True, models_dir, model_version, 8, dtype, shape) create_tf_modelfile(True, models_dir, model_version, 8, dtype, shape) if no_batch: create_tf_modelconfig(True, models_dir, model_version, 0, dtype, shape) create_tf_modelfile(True, models_dir, model_version, 0, dtype, shape) if FLAGS.netdef: create_netdef_modelconfig(models_dir, model_version, 8, dtype, shape) create_netdef_modelfile(models_dir, model_version, 8, dtype, shape) if no_batch: create_netdef_modelconfig(models_dir, model_version, 0, dtype, shape) create_netdef_modelfile(models_dir, model_version, 0, dtype, shape) if FLAGS.tensorrt: create_plan_modelconfig(models_dir, model_version, 8, dtype, shape + [1, 1]) create_plan_modelfile(models_dir, model_version, 8, dtype, shape + [1, 1]) if no_batch: create_plan_modelconfig(models_dir, model_version, 0, dtype, shape + [1, 1]) create_plan_modelfile(models_dir, model_version, 0, dtype, shape + [1, 1]) if FLAGS.onnx: create_onnx_modelconfig(models_dir, model_version, 8, dtype, shape) create_onnx_modelfile(models_dir, model_version, 8, dtype, shape) if no_batch: create_onnx_modelconfig(models_dir, model_version, 0, dtype, shape) create_onnx_modelfile(models_dir, model_version, 0, dtype, shape) if FLAGS.ensemble: for pair in emu.platform_types_and_validation(): if pair[0] == "plan": shape = shape + [1, 1] if not pair[1](dtype, dtype, dtype, shape, shape, shape): continue emu.create_sequence_ensemble_modelconfig( pair[0], models_dir, 8, model_version, shape, dtype) emu.create_sequence_ensemble_modelfile( pair[0], models_dir, 8, model_version, shape, dtype) if no_batch: emu.create_sequence_ensemble_modelconfig( pair[0], models_dir, 0, model_version, shape, dtype) emu.create_sequence_ensemble_modelfile( pair[0], models_dir, 0, model_version, shape, dtype)
def create_models( models_dir, input_dtype, output0_dtype, output1_dtype, input_shape, output0_shape, output1_shape, output0_label_cnt, version_policy=None): model_version = 1 # Create two models, one that supports batching with a max-batch # of 8, and one that does not with a max-batch of 0 if FLAGS.graphdef: # max-batch 8 create_graphdef_modelconfig( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_graphdef_modelfile( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) # max-batch 0 create_graphdef_modelconfig( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_graphdef_modelfile( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) if FLAGS.savedmodel: # max-batch 8 create_savedmodel_modelconfig( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_savedmodel_modelfile( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) # max-batch 0 create_savedmodel_modelconfig( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_savedmodel_modelfile( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) if FLAGS.netdef: # max-batch 8 create_netdef_modelconfig( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_netdef_modelfile( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) # max-batch 0 create_netdef_modelconfig( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_netdef_modelfile( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) if FLAGS.tensorrt: # max-batch 8 create_plan_modelconfig( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_plan_modelfile( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) # max-batch 0 create_plan_modelconfig( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_plan_modelfile( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) if FLAGS.onnx: # max-batch 8 create_onnx_modelconfig( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_onnx_modelfile( models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) # max-batch 0 create_onnx_modelconfig( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) create_onnx_modelfile( models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) if FLAGS.ensemble: for pair in emu.platform_types_and_validation(): if not pair[1](input_dtype, output0_dtype, output1_dtype, input_shape, output0_shape, output1_shape): continue if pair[0] == "plan": if len(input_shape) == 1: input_shape = (input_shape[0], 1, 1) if len(output0_shape) == 1: output0_shape = (output0_shape[0], 1, 1) if len(output1_shape) == 1: output1_shape = (output1_shape[0], 1, 1) # max-batch 8 emu.create_ensemble_modelconfig( pair[0], models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) emu.create_ensemble_modelfile( pair[0], models_dir, 8, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype) # max-batch 0 emu.create_ensemble_modelconfig( pair[0], models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy) emu.create_ensemble_modelfile( pair[0], models_dir, 0, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype)
create_plan_modelfile(FLAGS.models_dir, 0, 3, (16,1,1), (16,1,1), (16,1,1), vt, vt, vt, swap=True) if FLAGS.onnx: for vt in [np.float16, np.float32, np.int8, np.int16, np.int32]: create_onnx_modelfile(FLAGS.models_dir, 8, 2, (16,), (16,), (16,), vt, vt, vt, swap=True) create_onnx_modelfile(FLAGS.models_dir, 8, 3, (16,), (16,), (16,), vt, vt, vt, swap=True) create_onnx_modelfile(FLAGS.models_dir, 0, 2, (16,), (16,), (16,), vt, vt, vt, swap=True) create_onnx_modelfile(FLAGS.models_dir, 0, 3, (16,), (16,), (16,), vt, vt, vt, swap=True) if FLAGS.ensemble: for pair in emu.platform_types_and_validation(): shape = (16, 1, 1) if pair[0] == "plan" else (16,) for vt in [np.float16, np.float32, np.int8, np.int16, np.int32]: if not pair[1](vt, vt, vt, shape, shape, shape): continue emu.create_ensemble_modelfile(pair[0], FLAGS.models_dir, 8, 2, shape, shape, shape, vt, vt, vt, swap=True) emu.create_ensemble_modelfile(pair[0], FLAGS.models_dir, 8, 3, shape, shape, shape, vt, vt, vt, swap=True) emu.create_ensemble_modelfile(pair[0], FLAGS.models_dir, 0, 2, shape, shape, shape, vt, vt, vt, swap=True) emu.create_ensemble_modelfile(pair[0], FLAGS.models_dir, 0, 3, shape, shape, shape, vt, vt, vt, swap=True) # Tests with models that accept variable-shape input/output tensors if FLAGS.variable:
def create_models(models_dir, dtype, shape, no_batch=True): model_version = 1 if FLAGS.graphdef: create_tf_modelconfig(False, models_dir, model_version, 8, dtype, shape) create_tf_modelfile(False, models_dir, model_version, 8, dtype, shape) if no_batch: create_tf_modelconfig(False, models_dir, model_version, 0, dtype, shape) create_tf_modelfile(False, models_dir, model_version, 0, dtype, shape) if FLAGS.savedmodel: create_tf_modelconfig(True, models_dir, model_version, 8, dtype, shape) create_tf_modelfile(True, models_dir, model_version, 8, dtype, shape) if no_batch: create_tf_modelconfig(True, models_dir, model_version, 0, dtype, shape) create_tf_modelfile(True, models_dir, model_version, 0, dtype, shape) if FLAGS.tensorrt: suffix = [] if dtype == np.int8: suffix = [1, 1] create_plan_modelconfig(models_dir, model_version, 8, dtype, shape + suffix) create_plan_modelfile(models_dir, model_version, 8, dtype, shape + suffix) if no_batch: create_plan_modelconfig(models_dir, model_version, 0, dtype, shape + suffix) create_plan_modelfile(models_dir, model_version, 0, dtype, shape + suffix) if FLAGS.onnx: create_onnx_modelconfig(models_dir, model_version, 8, dtype, shape) create_onnx_modelfile(models_dir, model_version, 8, dtype, shape) if no_batch: create_onnx_modelconfig(models_dir, model_version, 0, dtype, shape) create_onnx_modelfile(models_dir, model_version, 0, dtype, shape) if FLAGS.libtorch: create_libtorch_modelconfig(models_dir, model_version, 8, dtype, shape) create_libtorch_modelfile(models_dir, model_version, 8, dtype, shape) if no_batch: create_libtorch_modelconfig(models_dir, model_version, 0, dtype, shape) create_libtorch_modelfile(models_dir, model_version, 0, dtype, shape) if FLAGS.ensemble: for pair in emu.platform_types_and_validation(): config_shape = shape if pair[0] == "plan" and dtype == np.int8: config_shape = shape + [1, 1] if not pair[1](dtype, dtype, dtype, config_shape, config_shape, config_shape): continue emu.create_sequence_ensemble_modelconfig(pair[0], models_dir, 8, model_version, config_shape, dtype) emu.create_sequence_ensemble_modelfile(pair[0], models_dir, 8, model_version, config_shape, dtype) if no_batch: emu.create_sequence_ensemble_modelconfig( pair[0], models_dir, 0, model_version, config_shape, dtype) emu.create_sequence_ensemble_modelfile(pair[0], models_dir, 0, model_version, config_shape, dtype)