def _get_fan_ensemble_schedule(cls, base_model_name, shape, model_dtype): # nop cannot handle STRING data type, fall back to simple if model_dtype == "TYPE_STRING": return SequenceEnsembleSchedule._get_simple_ensemble_schedule( base_model_name, shape, model_dtype) # Not a "fan" due to configuration of base sequence model # ensemble input -> nop -> sequence -> nop -> ensemble output nop_shape = fixed_to_variable_size(shape) schedule = ''' ensemble_scheduling {{ step [ {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "INPUT" }} input_map {{ key: "INPUT1" value: "INPUT" }} output_map {{ key: "OUTPUT0" value: "same_input" }} }}, {{ model_name: "{}" model_version: -1 input_map {{ key: "INPUT" value: "same_input" }} output_map {{ key: "OUTPUT" value: "same_output" }} }}, {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "same_output" }} input_map {{ key: "INPUT1" value: "same_output" }} output_map {{ key: "OUTPUT0" value: "OUTPUT" }} }} ] }} '''.format(model_dtype, tu.shape_to_dims_str(nop_shape), base_model_name, model_dtype, tu.shape_to_dims_str(nop_shape)) return schedule
def create_nop_modelconfig(models_dir, tensor_shape, tensor_dtype, tensor_model_shape=None): model_name = "nop_{}_{}".format(dtype_str(tensor_dtype), tu.shape_to_dims_str(tensor_shape)) # Make [] to [1]. # Note that this doesn't affect the naming ("nop_{}_" instead of "nop_{}_1") if len(tensor_shape) == 0: tensor_shape = [1] config_dir = models_dir + "/" + model_name config = create_general_modelconfig( model_name, "custom", 1024, repeat(tensor_dtype, 2), repeat(tensor_shape, 2), repeat(tensor_model_shape, 2), repeat(tensor_dtype, 2), repeat(tensor_shape, 2), repeat(tensor_model_shape, 2), repeat(None, 2), default_model_filename="libidentity.so", instance_group_str="instance_group [ { kind: KIND_CPU } ]") try: os.makedirs(config_dir) except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: cfile.write(config)
def _get_sequence_ensemble_schedule(cls, base_model_name, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype): # libtorch model uses other naming convention index_delimiter = "__" if "libtorch" in base_model_name else "" # ensemble input -> nop -> addsub -> ensemble output nop_input_shape = fixed_to_variable_size(input_shape) schedule = ''' ensemble_scheduling {{ step [ {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "INPUT0" }} input_map {{ key: "INPUT1" value: "INPUT1" }} output_map {{ key: "OUTPUT0" value: "same_input0" }} output_map {{ key: "OUTPUT1" value: "same_input1" }} }}, {{ model_name: "{}" model_version: -1 input_map {{ key: "INPUT{delimiter}0" value: "same_input0" }} input_map {{ key: "INPUT{delimiter}1" value: "same_input1" }} output_map {{ key: "OUTPUT{delimiter}0" value: "OUTPUT0" }} output_map {{ key: "OUTPUT{delimiter}1" value: "OUTPUT1" }} }} ] }} '''.format(input_dtype, tu.shape_to_dims_str(nop_input_shape), base_model_name, delimiter=index_delimiter) return schedule
def _get_sequence_ensemble_schedule(cls, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes, test_type): in_str = "tunnel_in_" if test_type == "reshape" else "" out_str = "tunnel_out_" if test_type == "reshape" else "" # ensemble reshaped input -> nop with another input only reshape -> # nop with output only reshape -> ensemble reshaped output steps = [] for idx in range(len(input_shapes)): steps.append(''' {{ model_name: "nop_{in_str}{type}_{shape}" model_version: -1 input_map {{ key: "INPUT0" value: "INPUT{idx}" }} input_map {{ key: "INPUT1" value: "INPUT{idx}" }} output_map {{ key: "OUTPUT0" value: "temp_{idx}" }} }}, {{ model_name: "nop_{out_str}{type}_{shape}" model_version: -1 input_map {{ key: "INPUT0" value: "temp_{idx}" }} input_map {{ key: "INPUT1" value: "temp_{idx}" }} output_map {{ key: "OUTPUT0" value: "OUTPUT{idx}" }} }} '''.format(type=np_to_model_dtype(dtype), in_str=in_str, out_str=out_str, idx=idx, shape=tu.shape_to_dims_str(input_model_shapes[idx]))) schedule = ''' ensemble_scheduling {{ step [ {} ] }} '''.format(",".join(steps)) return schedule
def create_nop_modelconfig(models_dir, tensor_shape, tensor_dtype): model_name = "nop_{}_{}".format(tensor_dtype, tu.shape_to_dims_str(tensor_shape)) config_dir = models_dir + "/" + model_name config = ''' name: "{model_name}" platform: "custom" max_batch_size: {batch_size} default_model_filename: "libidentity.so" input [ {{ name: "INPUT0" data_type: {dtype} dims: [ {dim} ] }}, {{ name: "INPUT1" data_type: {dtype} dims: [ {dim} ] }} ] output [ {{ name: "OUTPUT0" data_type: {dtype} dims: [ {dim} ] }}, {{ name: "OUTPUT1" data_type: {dtype} dims: [ {dim} ] }} ] instance_group [ {{ kind: KIND_CPU }} ] '''.format(model_name=model_name, dtype=tensor_dtype, batch_size=1024, dim=tu.shape_to_dims_str(tensor_shape)) try: os.makedirs(config_dir) except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: cfile.write(config)
def _get_sequence_ensemble_schedule(cls, base_model_name, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype): # ensemble input -> nop -> addsub -> ensemble output nop_input_shape = fixed_to_variable_size(input_shape) schedule = ''' ensemble_scheduling {{ step [ {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "INPUT0" }} input_map {{ key: "INPUT1" value: "INPUT1" }} output_map {{ key: "OUTPUT0" value: "same_input0" }} output_map {{ key: "OUTPUT1" value: "same_input1" }} }}, {{ model_name: "{}" model_version: -1 input_map {{ key: "INPUT0" value: "same_input0" }} input_map {{ key: "INPUT1" value: "same_input1" }} output_map {{ key: "OUTPUT0" value: "OUTPUT0" }} output_map {{ key: "OUTPUT1" value: "OUTPUT1" }} }} ] }} '''.format(input_dtype, tu.shape_to_dims_str(nop_input_shape), base_model_name) return schedule
def _get_sequence_ensemble_schedule(cls, base_model_name, shape, model_dtype): # nop cannot handle STRING data type, fall back to simple if model_dtype == "TYPE_STRING": return SequenceEnsembleSchedule._get_simple_ensemble_schedule( base_model_name, shape, model_dtype) # libtorch model uses other naming convention index_suffix = "__0" if "libtorch" in base_model_name else "" # ensemble input -> nop -> sequence -> ensemble output nop_input_shape = fixed_to_variable_size(shape) schedule = ''' ensemble_scheduling {{ step [ {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "INPUT" }} input_map {{ key: "INPUT1" value: "INPUT" }} output_map {{ key: "OUTPUT0" value: "same_input" }} }}, {{ model_name: "{}" model_version: -1 input_map {{ key: "INPUT{index}" value: "same_input" }} output_map {{ key: "OUTPUT{index}" value: "OUTPUT" }} }} ] }} '''.format(model_dtype, tu.shape_to_dims_str(nop_input_shape), base_model_name, index=index_suffix) return schedule
def _get_simple_ensemble_schedule(cls, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes, test_type): # ensemble reshaped input -> nop with reshaped tensor shape -> ensemble # reshaped output (actual ensemble input/output is not visible in schedule) steps = [] for idx in range(len(input_shapes)): steps.append(''' {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "INPUT{}" }} input_map {{ key: "INPUT1" value: "INPUT{}" }} output_map {{ key: "OUTPUT0" value: "OUTPUT{}" }} }} '''.format(np_to_model_dtype(dtype), tu.shape_to_dims_str(input_model_shapes[idx]), idx, idx, idx)) schedule = ''' ensemble_scheduling {{ step [ {} ] }} '''.format(",".join(steps)) return schedule
def create_ensemble_modelconfig( base_model, models_dir, max_batch, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, output0_label_cnt, version_policy): # No validation as long as the base model supports the type and shape # Unpack version policy version_policy_str = "{ latest { num_versions: 1 }}" if version_policy is not None: type, val = version_policy if type == 'latest': version_policy_str = "{{ latest {{ num_versions: {} }}}}".format(val) elif type == 'specific': version_policy_str = "{{ specific {{ versions: {} }}}}".format(val) else: version_policy_str = "{ all { }}" input_model_dtype = np_to_model_dtype(input_dtype) output0_model_dtype = np_to_model_dtype(output0_dtype) output1_model_dtype = np_to_model_dtype(output1_dtype) for ensemble_type in BASIC_ENSEMBLE_TYPES: # Use a different model name for the non-batching variant ensemble_model_name = "{}_{}{}".format(ensemble_type, base_model, "_nobatch" if max_batch == 0 else "") model_name = tu.get_model_name(ensemble_model_name, input_dtype, output0_dtype, output1_dtype) base_model_name = tu.get_model_name("{}{}".format(base_model, "_nobatch" if max_batch == 0 else ""), input_dtype, output0_dtype, output1_dtype) ensemble_schedule = EnsembleSchedule(ensemble_type).get_schedule( base_model_name, input_shape, output0_shape, output1_shape, input_model_dtype, output0_model_dtype, output1_model_dtype) config_dir = models_dir + "/" + model_name config = ''' name: "{}" platform: "ensemble" max_batch_size: {} version_policy: {} input [ {{ name: "INPUT0" data_type: {} dims: [ {} ] }}, {{ name: "INPUT1" data_type: {} dims: [ {} ] }} ] output [ {{ name: "OUTPUT0" data_type: {} dims: [ {} ] label_filename: "output0_labels.txt" }}, {{ name: "OUTPUT1" data_type: {} dims: [ {} ] }} ] {} '''.format(model_name, max_batch, version_policy_str, input_model_dtype, tu.shape_to_dims_str(input_shape), input_model_dtype, tu.shape_to_dims_str(input_shape), output0_model_dtype, tu.shape_to_dims_str(output0_shape), output1_model_dtype, tu.shape_to_dims_str(output1_shape), ensemble_schedule) try: os.makedirs(config_dir) except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: cfile.write(config) with open(config_dir + "/output0_labels.txt", "w") as lfile: for l in range(output0_label_cnt): lfile.write("label" + str(l) + "\n")
def _get_fan_ensemble_steps(cls, base_model_name, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype): # ensemble input -> nop -> addsub -> # nop (fan out, one output send to one nop) -> ensemble output (fan in) nop_input_shape = fixed_to_variable_size(input_shape) nop_output0_shape = fixed_to_variable_size(output0_shape) nop_output1_shape = fixed_to_variable_size(output1_shape) steps = ''' ensemble_scheduling {{ step [ {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "INPUT0" }} input_map {{ key: "INPUT1" value: "INPUT1" }} output_map {{ key: "OUTPUT0" value: "same_input0" }} output_map {{ key: "OUTPUT1" value: "same_input1" }} }}, {{ model_name: "{}" model_version: -1 input_map {{ key: "INPUT0" value: "same_input0" }} input_map {{ key: "INPUT1" value: "same_input1" }} output_map {{ key: "OUTPUT0" value: "same_output0" }} output_map {{ key: "OUTPUT1" value: "same_output1" }} }}, {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "same_output0" }} input_map {{ key: "INPUT1" value: "same_output0" }} output_map {{ key: "OUTPUT0" value: "OUTPUT0" }} }}, {{ model_name: "nop_{}_{}" model_version: -1 input_map {{ key: "INPUT0" value: "same_output1" }} input_map {{ key: "INPUT1" value: "same_output1" }} output_map {{ key: "OUTPUT1" value: "OUTPUT1" }} }} ] }} '''.format(input_dtype, tu.shape_to_dims_str(nop_input_shape), base_model_name, output0_dtype, tu.shape_to_dims_str(nop_output0_shape), output1_dtype, tu.shape_to_dims_str(nop_output1_shape)) return steps
def create_general_modelconfig(model_name, platform, max_batch, input_dtypes, input_shapes, input_model_shapes, output_dtypes, output_shapes, output_model_shapes, label_filenames, version_policy=None, default_model_filename=None, instance_group_str="", force_tensor_number_suffix=False): assert len(input_dtypes) == len(input_shapes) assert len(input_model_shapes) == len(input_shapes) assert len(output_dtypes) == len(output_shapes) assert len(output_model_shapes) == len(output_shapes) assert len(label_filenames) == len(output_shapes) # Unpack version policy version_policy_str = "{ latest { num_versions: 1 }}" if version_policy is not None: type, val = version_policy if type == 'latest': version_policy_str = "{{ latest {{ num_versions: {} }}}}".format( val) elif type == 'specific': version_policy_str = "{{ specific {{ versions: {} }}}}".format(val) else: version_policy_str = "{ all { }}" default_model_filename_str = "" if default_model_filename is not None: default_model_filename_str = 'default_model_filename: "{}"'.format( default_model_filename) config = ''' name: "{}" platform: "{}" max_batch_size: {} version_policy : {} {} {} '''.format(model_name, platform, max_batch, version_policy_str, default_model_filename_str, instance_group_str) for idx in range(len(input_dtypes)): idx_str = "" if len(input_dtypes) != 1 or force_tensor_number_suffix: idx_str = str(idx) config += ''' input [ {{ name: "INPUT{}" data_type: {} dims: [ {} ] {} }} ]'''.format(idx_str, dtype_str(input_dtypes[idx]), tu.shape_to_dims_str(input_shapes[idx]), reshape_str(input_shapes[idx], input_model_shapes[idx])) for idx in range(len(output_dtypes)): idx_str = "" if len(input_dtypes) != 1 or force_tensor_number_suffix: idx_str = str(idx) config += ''' output [ {{ name: "OUTPUT{}" data_type: {} dims: [ {} ] {} {} }} ]'''.format(idx_str, dtype_str(output_dtypes[idx]), tu.shape_to_dims_str(output_shapes[idx]), reshape_str(output_shapes[idx], output_model_shapes[idx]), label_str(label_filenames[idx])) return config
def create_nop_tunnel_modelconfig(models_dir, tensor_shape, tensor_dtype): # Must be fixed size in_model_name = "nop_tunnel_in_{}_{}".format( dtype_str(tensor_dtype), tu.shape_to_dims_str(tensor_shape)) out_model_name = "nop_tunnel_out_{}_{}".format( dtype_str(tensor_dtype), tu.shape_to_dims_str(tensor_shape)) # Make [] to [1]. # Note that this doesn't affect the naming ("nop_{}_" instead of "nop_{}_1") if len(tensor_shape) == 0: tensor_shape = [1] internal_shape = 1 for dim in tensor_shape: if dim < 0: raise Exception( "Must specify fixed size input / output for nop tunnel") internal_shape *= dim # Tunnel in nop (reshape to one dimension) config_dir = models_dir + "/" + in_model_name config = create_general_modelconfig( in_model_name, "custom", 1024, repeat(tensor_dtype, 2), repeat(tensor_shape, 2), repeat([internal_shape], 2), repeat(tensor_dtype, 2), repeat([internal_shape], 2), repeat(None, 2), repeat(None, 2), default_model_filename="libidentity.so", instance_group_str="instance_group [ { kind: KIND_CPU } ]") try: os.makedirs(config_dir) except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: cfile.write(config) # Tunnel out nop (reshape back to original shape) config_dir = models_dir + "/" + out_model_name config = create_general_modelconfig( out_model_name, "custom", 1024, repeat(tensor_dtype, 2), repeat([internal_shape], 2), repeat(tensor_shape, 2), repeat(tensor_dtype, 2), repeat(tensor_shape, 2), repeat(None, 2), repeat(None, 2), default_model_filename="libidentity.so", instance_group_str="instance_group [ { kind: KIND_CPU } ]") try: os.makedirs(config_dir) except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: cfile.write(config)
def _get_fan_ensemble_schedule(cls, dtype, input_shapes, input_model_shapes, output_shapes, output_model_shapes, test_type): # Note that the simple and sequence test already test "fan" in some # degree, because there is no direct match from nop input/output # like what is in addsub-like ensemble. # # ensemble reshaped input -> nop with another input only reshape -> # nop with variable size -> nop with output only reshape -> # ensemble reshaped output in_str = "" out_str = "" intermediate_shapes = input_model_shapes if test_type == "reshape": in_str = "tunnel_in_" out_str = "tunnel_out_" intermediate_shapes = [[-1]] * len(input_model_shapes) steps = [] for idx in range(len(input_shapes)): steps.append(''' {{ model_name: "nop_{in_str}{type}_{shape}" model_version: -1 input_map {{ key: "INPUT0" value: "INPUT{idx}" }} input_map {{ key: "INPUT1" value: "INPUT{idx}" }} output_map {{ key: "OUTPUT0" value: "temp_in_{idx}" }} }}, {{ model_name: "nop_{type}_{intermediate_shape}" model_version: -1 input_map {{ key: "INPUT0" value: "temp_in_{idx}" }} input_map {{ key: "INPUT1" value: "temp_in_{idx}" }} output_map {{ key: "OUTPUT0" value: "temp_out_{idx}" }} }}, {{ model_name: "nop_{out_str}{type}_{shape}" model_version: -1 input_map {{ key: "INPUT0" value: "temp_out_{idx}" }} input_map {{ key: "INPUT1" value: "temp_out_{idx}" }} output_map {{ key: "OUTPUT0" value: "OUTPUT{idx}" }} }} '''.format(type=np_to_model_dtype(dtype), in_str=in_str, out_str=out_str, intermediate_shape=tu.shape_to_dims_str(intermediate_shapes[idx]), idx=idx, shape=tu.shape_to_dims_str(input_model_shapes[idx]))) schedule = ''' ensemble_scheduling {{ step [ {} ] }} '''.format(",".join(steps)) return schedule
def reshape_str(shape, model_shape): if model_shape is None or shape == model_shape: return "" return "reshape: {{ shape: [ {} ] }}".format( tu.shape_to_dims_str(model_shape))
def create_plan_modelconfig(models_dir, max_batch, model_version, input_shape, output0_shape, output1_shape, input_dtype, output0_dtype, output1_dtype, input_memory_format, output_memory_format, version_policy): if not tu.validate_for_trt_model(input_dtype, output0_dtype, output1_dtype, input_shape, output0_shape, output1_shape): return # Unpack version policy version_policy_str = "{ latest { num_versions: 1 }}" if version_policy is not None: type, val = version_policy if type == 'latest': version_policy_str = "{{ latest {{ num_versions: {} }}}}".format( val) elif type == 'specific': version_policy_str = "{{ specific {{ versions: {} }}}}".format(val) else: version_policy_str = "{ all { }}" # Use a different model name for different kinds of models base_name = "plan_nobatch" if max_batch == 0 else "plan" base_name += "_" + trt_format_to_string( input_memory_format) + "_" + trt_format_to_string(output_memory_format) model_name = tu.get_model_name(base_name, input_dtype, output0_dtype, output1_dtype) config_dir = models_dir + "/" + model_name if -1 in input_shape: profile_index = 0 config = ''' name: "{}" platform: "tensorrt_plan" max_batch_size: {} version_policy: {} input [ {{ name: "INPUT0" data_type: {} dims: [ {} ] }}, {{ name: "INPUT1" data_type: {} dims: [ {} ] }} ] output [ {{ name: "OUTPUT0" data_type: {} dims: [ {} ] }}, {{ name: "OUTPUT1" data_type: {} dims: [ {} ] }} ] instance_group [ {{ profile:"{}" }} ] '''.format(model_name, max_batch, version_policy_str, np_to_model_dtype(input_dtype), tu.shape_to_dims_str(input_shape), np_to_model_dtype(input_dtype), tu.shape_to_dims_str(input_shape), np_to_model_dtype(output0_dtype), tu.shape_to_dims_str(output0_shape), np_to_model_dtype(output1_dtype), tu.shape_to_dims_str(output1_shape), profile_index) else: config = ''' name: "{}" platform: "tensorrt_plan" max_batch_size: {} version_policy: {} input [ {{ name: "INPUT0" data_type: {} dims: [ {} ] }}, {{ name: "INPUT1" data_type: {} dims: [ {} ] }} ] output [ {{ name: "OUTPUT0" data_type: {} dims: [ {} ] }}, {{ name: "OUTPUT1" data_type: {} dims: [ {} ] }} ] '''.format(model_name, max_batch, version_policy_str, np_to_model_dtype(input_dtype), tu.shape_to_dims_str(input_shape), np_to_model_dtype(input_dtype), tu.shape_to_dims_str(input_shape), np_to_model_dtype(output0_dtype), tu.shape_to_dims_str(output0_shape), np_to_model_dtype(output1_dtype), tu.shape_to_dims_str(output1_shape)) try: os.makedirs(config_dir) except OSError as ex: pass # ignore existing dir with open(config_dir + "/config.pbtxt", "w") as cfile: cfile.write(config)