def _get_fan_ensemble_schedule(cls, base_model_name, shape, model_dtype):
# nop cannot handle STRING data type, fall back to simple
if model_dtype == "TYPE_STRING":
return SequenceEnsembleSchedule._get_simple_ensemble_schedule(
base_model_name, shape, model_dtype)
# Not a "fan" due to configuration of base sequence model
# ensemble input -> nop -> sequence -> nop -> ensemble output
nop_shape = fixed_to_variable_size(shape)
schedule = '''
ensemble_scheduling {{
step [
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "INPUT"
}}
input_map {{
key: "INPUT1"
value: "INPUT"
}}
output_map {{
key: "OUTPUT0"
value: "same_input"
}}
}},
{{
model_name: "{}"
model_version: -1
input_map {{
key: "INPUT"
value: "same_input"
}}
output_map {{
key: "OUTPUT"
value: "same_output"
}}
}},
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "same_output"
}}
input_map {{
key: "INPUT1"
value: "same_output"
}}
output_map {{
key: "OUTPUT0"
value: "OUTPUT"
}}
}}
]
}}
'''.format(model_dtype, tu.shape_to_dims_str(nop_shape), base_model_name,
model_dtype, tu.shape_to_dims_str(nop_shape))
return schedule
def create_nop_modelconfig(models_dir,
tensor_shape,
tensor_dtype,
tensor_model_shape=None):
model_name = "nop_{}_{}".format(dtype_str(tensor_dtype),
tu.shape_to_dims_str(tensor_shape))
# Make [] to [1].
# Note that this doesn't affect the naming ("nop_{}_" instead of "nop_{}_1")
if len(tensor_shape) == 0:
tensor_shape = [1]
config_dir = models_dir + "/" + model_name
config = create_general_modelconfig(
model_name,
"custom",
1024,
repeat(tensor_dtype, 2),
repeat(tensor_shape, 2),
repeat(tensor_model_shape, 2),
repeat(tensor_dtype, 2),
repeat(tensor_shape, 2),
repeat(tensor_model_shape, 2),
repeat(None, 2),
default_model_filename="libidentity.so",
instance_group_str="instance_group [ { kind: KIND_CPU } ]")
try:
os.makedirs(config_dir)
except OSError as ex:
pass # ignore existing dir
with open(config_dir + "/config.pbtxt", "w") as cfile:
cfile.write(config)
def _get_sequence_ensemble_schedule(cls, base_model_name, input_shape,
output0_shape, output1_shape,
input_dtype, output0_dtype,
output1_dtype):
# libtorch model uses other naming convention
index_delimiter = "__" if "libtorch" in base_model_name else ""
# ensemble input -> nop -> addsub -> ensemble output
nop_input_shape = fixed_to_variable_size(input_shape)
schedule = '''
ensemble_scheduling {{
step [
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "INPUT0"
}}
input_map {{
key: "INPUT1"
value: "INPUT1"
}}
output_map {{
key: "OUTPUT0"
value: "same_input0"
}}
output_map {{
key: "OUTPUT1"
value: "same_input1"
}}
}},
{{
model_name: "{}"
model_version: -1
input_map {{
key: "INPUT{delimiter}0"
value: "same_input0"
}}
input_map {{
key: "INPUT{delimiter}1"
value: "same_input1"
}}
output_map {{
key: "OUTPUT{delimiter}0"
value: "OUTPUT0"
}}
output_map {{
key: "OUTPUT{delimiter}1"
value: "OUTPUT1"
}}
}}
]
}}
'''.format(input_dtype,
tu.shape_to_dims_str(nop_input_shape),
base_model_name,
delimiter=index_delimiter)
return schedule
def _get_sequence_ensemble_schedule(cls, dtype, input_shapes,
input_model_shapes, output_shapes,
output_model_shapes, test_type):
in_str = "tunnel_in_" if test_type == "reshape" else ""
out_str = "tunnel_out_" if test_type == "reshape" else ""
# ensemble reshaped input -> nop with another input only reshape ->
# nop with output only reshape -> ensemble reshaped output
steps = []
for idx in range(len(input_shapes)):
steps.append('''
{{
model_name: "nop_{in_str}{type}_{shape}"
model_version: -1
input_map {{
key: "INPUT0"
value: "INPUT{idx}"
}}
input_map {{
key: "INPUT1"
value: "INPUT{idx}"
}}
output_map {{
key: "OUTPUT0"
value: "temp_{idx}"
}}
}},
{{
model_name: "nop_{out_str}{type}_{shape}"
model_version: -1
input_map {{
key: "INPUT0"
value: "temp_{idx}"
}}
input_map {{
key: "INPUT1"
value: "temp_{idx}"
}}
output_map {{
key: "OUTPUT0"
value: "OUTPUT{idx}"
}}
}}
'''.format(type=np_to_model_dtype(dtype),
in_str=in_str,
out_str=out_str,
idx=idx,
shape=tu.shape_to_dims_str(input_model_shapes[idx])))
schedule = '''
ensemble_scheduling {{
step [
{}
]
}}
'''.format(",".join(steps))
return schedule
def create_nop_modelconfig(models_dir, tensor_shape, tensor_dtype):
model_name = "nop_{}_{}".format(tensor_dtype, tu.shape_to_dims_str(tensor_shape))
config_dir = models_dir + "/" + model_name
config = '''
name: "{model_name}"
platform: "custom"
max_batch_size: {batch_size}
default_model_filename: "libidentity.so"
input [
{{
name: "INPUT0"
data_type: {dtype}
dims: [ {dim} ]
}},
{{
name: "INPUT1"
data_type: {dtype}
dims: [ {dim} ]
}}
]
output [
{{
name: "OUTPUT0"
data_type: {dtype}
dims: [ {dim} ]
}},
{{
name: "OUTPUT1"
data_type: {dtype}
dims: [ {dim} ]
}}
]
instance_group [ {{ kind: KIND_CPU }} ]
'''.format(model_name=model_name, dtype=tensor_dtype,
batch_size=1024, dim=tu.shape_to_dims_str(tensor_shape))
try:
os.makedirs(config_dir)
except OSError as ex:
pass # ignore existing dir
with open(config_dir + "/config.pbtxt", "w") as cfile:
cfile.write(config)
def _get_sequence_ensemble_schedule(cls, base_model_name, input_shape,
output0_shape, output1_shape,
input_dtype, output0_dtype,
output1_dtype):
# ensemble input -> nop -> addsub -> ensemble output
nop_input_shape = fixed_to_variable_size(input_shape)
schedule = '''
ensemble_scheduling {{
step [
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "INPUT0"
}}
input_map {{
key: "INPUT1"
value: "INPUT1"
}}
output_map {{
key: "OUTPUT0"
value: "same_input0"
}}
output_map {{
key: "OUTPUT1"
value: "same_input1"
}}
}},
{{
model_name: "{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "same_input0"
}}
input_map {{
key: "INPUT1"
value: "same_input1"
}}
output_map {{
key: "OUTPUT0"
value: "OUTPUT0"
}}
output_map {{
key: "OUTPUT1"
value: "OUTPUT1"
}}
}}
]
}}
'''.format(input_dtype, tu.shape_to_dims_str(nop_input_shape), base_model_name)
return schedule
def _get_sequence_ensemble_schedule(cls, base_model_name, shape,
model_dtype):
# nop cannot handle STRING data type, fall back to simple
if model_dtype == "TYPE_STRING":
return SequenceEnsembleSchedule._get_simple_ensemble_schedule(
base_model_name, shape, model_dtype)
# libtorch model uses other naming convention
index_suffix = "__0" if "libtorch" in base_model_name else ""
# ensemble input -> nop -> sequence -> ensemble output
nop_input_shape = fixed_to_variable_size(shape)
schedule = '''
ensemble_scheduling {{
step [
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "INPUT"
}}
input_map {{
key: "INPUT1"
value: "INPUT"
}}
output_map {{
key: "OUTPUT0"
value: "same_input"
}}
}},
{{
model_name: "{}"
model_version: -1
input_map {{
key: "INPUT{index}"
value: "same_input"
}}
output_map {{
key: "OUTPUT{index}"
value: "OUTPUT"
}}
}}
]
}}
'''.format(model_dtype,
tu.shape_to_dims_str(nop_input_shape),
base_model_name,
index=index_suffix)
return schedule
def _get_simple_ensemble_schedule(cls, dtype, input_shapes,
input_model_shapes, output_shapes,
output_model_shapes, test_type):
# ensemble reshaped input -> nop with reshaped tensor shape -> ensemble
# reshaped output (actual ensemble input/output is not visible in schedule)
steps = []
for idx in range(len(input_shapes)):
steps.append('''
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "INPUT{}"
}}
input_map {{
key: "INPUT1"
value: "INPUT{}"
}}
output_map {{
key: "OUTPUT0"
value: "OUTPUT{}"
}}
}}
'''.format(np_to_model_dtype(dtype),
tu.shape_to_dims_str(input_model_shapes[idx]), idx, idx, idx))
schedule = '''
ensemble_scheduling {{
step [
{}
]
}}
'''.format(",".join(steps))
return schedule
def create_ensemble_modelconfig(
base_model, models_dir, max_batch, model_version,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype,
output0_label_cnt, version_policy):
# No validation as long as the base model supports the type and shape
# Unpack version policy
version_policy_str = "{ latest { num_versions: 1 }}"
if version_policy is not None:
type, val = version_policy
if type == 'latest':
version_policy_str = "{{ latest {{ num_versions: {} }}}}".format(val)
elif type == 'specific':
version_policy_str = "{{ specific {{ versions: {} }}}}".format(val)
else:
version_policy_str = "{ all { }}"
input_model_dtype = np_to_model_dtype(input_dtype)
output0_model_dtype = np_to_model_dtype(output0_dtype)
output1_model_dtype = np_to_model_dtype(output1_dtype)
for ensemble_type in BASIC_ENSEMBLE_TYPES:
# Use a different model name for the non-batching variant
ensemble_model_name = "{}_{}{}".format(ensemble_type, base_model, "_nobatch" if max_batch == 0 else "")
model_name = tu.get_model_name(ensemble_model_name,
input_dtype, output0_dtype, output1_dtype)
base_model_name = tu.get_model_name("{}{}".format(base_model, "_nobatch" if max_batch == 0 else ""),
input_dtype, output0_dtype, output1_dtype)
ensemble_schedule = EnsembleSchedule(ensemble_type).get_schedule(
base_model_name, input_shape, output0_shape,
output1_shape, input_model_dtype,
output0_model_dtype, output1_model_dtype)
config_dir = models_dir + "/" + model_name
config = '''
name: "{}"
platform: "ensemble"
max_batch_size: {}
version_policy: {}
input [
{{
name: "INPUT0"
data_type: {}
dims: [ {} ]
}},
{{
name: "INPUT1"
data_type: {}
dims: [ {} ]
}}
]
output [
{{
name: "OUTPUT0"
data_type: {}
dims: [ {} ]
label_filename: "output0_labels.txt"
}},
{{
name: "OUTPUT1"
data_type: {}
dims: [ {} ]
}}
]
{}
'''.format(model_name, max_batch, version_policy_str,
input_model_dtype, tu.shape_to_dims_str(input_shape),
input_model_dtype, tu.shape_to_dims_str(input_shape),
output0_model_dtype, tu.shape_to_dims_str(output0_shape),
output1_model_dtype, tu.shape_to_dims_str(output1_shape),
ensemble_schedule)
try:
os.makedirs(config_dir)
except OSError as ex:
pass # ignore existing dir
with open(config_dir + "/config.pbtxt", "w") as cfile:
cfile.write(config)
with open(config_dir + "/output0_labels.txt", "w") as lfile:
for l in range(output0_label_cnt):
lfile.write("label" + str(l) + "\n")
def _get_fan_ensemble_steps(cls, base_model_name,
input_shape, output0_shape, output1_shape,
input_dtype, output0_dtype, output1_dtype):
# ensemble input -> nop -> addsub ->
# nop (fan out, one output send to one nop) -> ensemble output (fan in)
nop_input_shape = fixed_to_variable_size(input_shape)
nop_output0_shape = fixed_to_variable_size(output0_shape)
nop_output1_shape = fixed_to_variable_size(output1_shape)
steps = '''
ensemble_scheduling {{
step [
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "INPUT0"
}}
input_map {{
key: "INPUT1"
value: "INPUT1"
}}
output_map {{
key: "OUTPUT0"
value: "same_input0"
}}
output_map {{
key: "OUTPUT1"
value: "same_input1"
}}
}},
{{
model_name: "{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "same_input0"
}}
input_map {{
key: "INPUT1"
value: "same_input1"
}}
output_map {{
key: "OUTPUT0"
value: "same_output0"
}}
output_map {{
key: "OUTPUT1"
value: "same_output1"
}}
}},
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "same_output0"
}}
input_map {{
key: "INPUT1"
value: "same_output0"
}}
output_map {{
key: "OUTPUT0"
value: "OUTPUT0"
}}
}},
{{
model_name: "nop_{}_{}"
model_version: -1
input_map {{
key: "INPUT0"
value: "same_output1"
}}
input_map {{
key: "INPUT1"
value: "same_output1"
}}
output_map {{
key: "OUTPUT1"
value: "OUTPUT1"
}}
}}
]
}}
'''.format(input_dtype, tu.shape_to_dims_str(nop_input_shape), base_model_name,
output0_dtype, tu.shape_to_dims_str(nop_output0_shape),
output1_dtype, tu.shape_to_dims_str(nop_output1_shape))
return steps
def create_general_modelconfig(model_name,
platform,
max_batch,
input_dtypes,
input_shapes,
input_model_shapes,
output_dtypes,
output_shapes,
output_model_shapes,
label_filenames,
version_policy=None,
default_model_filename=None,
instance_group_str="",
force_tensor_number_suffix=False):
assert len(input_dtypes) == len(input_shapes)
assert len(input_model_shapes) == len(input_shapes)
assert len(output_dtypes) == len(output_shapes)
assert len(output_model_shapes) == len(output_shapes)
assert len(label_filenames) == len(output_shapes)
# Unpack version policy
version_policy_str = "{ latest { num_versions: 1 }}"
if version_policy is not None:
type, val = version_policy
if type == 'latest':
version_policy_str = "{{ latest {{ num_versions: {} }}}}".format(
val)
elif type == 'specific':
version_policy_str = "{{ specific {{ versions: {} }}}}".format(val)
else:
version_policy_str = "{ all { }}"
default_model_filename_str = ""
if default_model_filename is not None:
default_model_filename_str = 'default_model_filename: "{}"'.format(
default_model_filename)
config = '''
name: "{}"
platform: "{}"
max_batch_size: {}
version_policy : {}
{}
{}
'''.format(model_name, platform, max_batch, version_policy_str,
default_model_filename_str, instance_group_str)
for idx in range(len(input_dtypes)):
idx_str = ""
if len(input_dtypes) != 1 or force_tensor_number_suffix:
idx_str = str(idx)
config += '''
input [
{{
name: "INPUT{}"
data_type: {}
dims: [ {} ]
{}
}}
]'''.format(idx_str, dtype_str(input_dtypes[idx]),
tu.shape_to_dims_str(input_shapes[idx]),
reshape_str(input_shapes[idx], input_model_shapes[idx]))
for idx in range(len(output_dtypes)):
idx_str = ""
if len(input_dtypes) != 1 or force_tensor_number_suffix:
idx_str = str(idx)
config += '''
output [
{{
name: "OUTPUT{}"
data_type: {}
dims: [ {} ]
{}
{}
}}
]'''.format(idx_str, dtype_str(output_dtypes[idx]),
tu.shape_to_dims_str(output_shapes[idx]),
reshape_str(output_shapes[idx], output_model_shapes[idx]),
label_str(label_filenames[idx]))
return config
def create_nop_tunnel_modelconfig(models_dir, tensor_shape, tensor_dtype):
# Must be fixed size
in_model_name = "nop_tunnel_in_{}_{}".format(
dtype_str(tensor_dtype), tu.shape_to_dims_str(tensor_shape))
out_model_name = "nop_tunnel_out_{}_{}".format(
dtype_str(tensor_dtype), tu.shape_to_dims_str(tensor_shape))
# Make [] to [1].
# Note that this doesn't affect the naming ("nop_{}_" instead of "nop_{}_1")
if len(tensor_shape) == 0:
tensor_shape = [1]
internal_shape = 1
for dim in tensor_shape:
if dim < 0:
raise Exception(
"Must specify fixed size input / output for nop tunnel")
internal_shape *= dim
# Tunnel in nop (reshape to one dimension)
config_dir = models_dir + "/" + in_model_name
config = create_general_modelconfig(
in_model_name,
"custom",
1024,
repeat(tensor_dtype, 2),
repeat(tensor_shape, 2),
repeat([internal_shape], 2),
repeat(tensor_dtype, 2),
repeat([internal_shape], 2),
repeat(None, 2),
repeat(None, 2),
default_model_filename="libidentity.so",
instance_group_str="instance_group [ { kind: KIND_CPU } ]")
try:
os.makedirs(config_dir)
except OSError as ex:
pass # ignore existing dir
with open(config_dir + "/config.pbtxt", "w") as cfile:
cfile.write(config)
# Tunnel out nop (reshape back to original shape)
config_dir = models_dir + "/" + out_model_name
config = create_general_modelconfig(
out_model_name,
"custom",
1024,
repeat(tensor_dtype, 2),
repeat([internal_shape], 2),
repeat(tensor_shape, 2),
repeat(tensor_dtype, 2),
repeat(tensor_shape, 2),
repeat(None, 2),
repeat(None, 2),
default_model_filename="libidentity.so",
instance_group_str="instance_group [ { kind: KIND_CPU } ]")
try:
os.makedirs(config_dir)
except OSError as ex:
pass # ignore existing dir
with open(config_dir + "/config.pbtxt", "w") as cfile:
cfile.write(config)
def _get_fan_ensemble_schedule(cls, dtype, input_shapes,
input_model_shapes, output_shapes,
output_model_shapes, test_type):
# Note that the simple and sequence test already test "fan" in some
# degree, because there is no direct match from nop input/output
# like what is in addsub-like ensemble.
#
# ensemble reshaped input -> nop with another input only reshape ->
# nop with variable size -> nop with output only reshape ->
# ensemble reshaped output
in_str = ""
out_str = ""
intermediate_shapes = input_model_shapes
if test_type == "reshape":
in_str = "tunnel_in_"
out_str = "tunnel_out_"
intermediate_shapes = [[-1]] * len(input_model_shapes)
steps = []
for idx in range(len(input_shapes)):
steps.append('''
{{
model_name: "nop_{in_str}{type}_{shape}"
model_version: -1
input_map {{
key: "INPUT0"
value: "INPUT{idx}"
}}
input_map {{
key: "INPUT1"
value: "INPUT{idx}"
}}
output_map {{
key: "OUTPUT0"
value: "temp_in_{idx}"
}}
}},
{{
model_name: "nop_{type}_{intermediate_shape}"
model_version: -1
input_map {{
key: "INPUT0"
value: "temp_in_{idx}"
}}
input_map {{
key: "INPUT1"
value: "temp_in_{idx}"
}}
output_map {{
key: "OUTPUT0"
value: "temp_out_{idx}"
}}
}},
{{
model_name: "nop_{out_str}{type}_{shape}"
model_version: -1
input_map {{
key: "INPUT0"
value: "temp_out_{idx}"
}}
input_map {{
key: "INPUT1"
value: "temp_out_{idx}"
}}
output_map {{
key: "OUTPUT0"
value: "OUTPUT{idx}"
}}
}}
'''.format(type=np_to_model_dtype(dtype),
in_str=in_str,
out_str=out_str,
intermediate_shape=tu.shape_to_dims_str(intermediate_shapes[idx]),
idx=idx,
shape=tu.shape_to_dims_str(input_model_shapes[idx])))
schedule = '''
ensemble_scheduling {{
step [
{}
]
}}
'''.format(",".join(steps))
return schedule
def reshape_str(shape, model_shape):
if model_shape is None or shape == model_shape:
return ""
return "reshape: {{ shape: [ {} ] }}".format(
tu.shape_to_dims_str(model_shape))
def create_plan_modelconfig(models_dir, max_batch, model_version, input_shape,
output0_shape, output1_shape, input_dtype,
output0_dtype, output1_dtype, input_memory_format,
output_memory_format, version_policy):
if not tu.validate_for_trt_model(input_dtype, output0_dtype, output1_dtype,
input_shape, output0_shape,
output1_shape):
return
# Unpack version policy
version_policy_str = "{ latest { num_versions: 1 }}"
if version_policy is not None:
type, val = version_policy
if type == 'latest':
version_policy_str = "{{ latest {{ num_versions: {} }}}}".format(
val)
elif type == 'specific':
version_policy_str = "{{ specific {{ versions: {} }}}}".format(val)
else:
version_policy_str = "{ all { }}"
# Use a different model name for different kinds of models
base_name = "plan_nobatch" if max_batch == 0 else "plan"
base_name += "_" + trt_format_to_string(
input_memory_format) + "_" + trt_format_to_string(output_memory_format)
model_name = tu.get_model_name(base_name, input_dtype, output0_dtype,
output1_dtype)
config_dir = models_dir + "/" + model_name
if -1 in input_shape:
profile_index = 0
config = '''
name: "{}"
platform: "tensorrt_plan"
max_batch_size: {}
version_policy: {}
input [
{{
name: "INPUT0"
data_type: {}
dims: [ {} ]
}},
{{
name: "INPUT1"
data_type: {}
dims: [ {} ]
}}
]
output [
{{
name: "OUTPUT0"
data_type: {}
dims: [ {} ]
}},
{{
name: "OUTPUT1"
data_type: {}
dims: [ {} ]
}}
]
instance_group [
{{
profile:"{}"
}}
]
'''.format(model_name, max_batch, version_policy_str,
np_to_model_dtype(input_dtype), tu.shape_to_dims_str(input_shape),
np_to_model_dtype(input_dtype), tu.shape_to_dims_str(input_shape),
np_to_model_dtype(output0_dtype),
tu.shape_to_dims_str(output0_shape),
np_to_model_dtype(output1_dtype),
tu.shape_to_dims_str(output1_shape), profile_index)
else:
config = '''
name: "{}"
platform: "tensorrt_plan"
max_batch_size: {}
version_policy: {}
input [
{{
name: "INPUT0"
data_type: {}
dims: [ {} ]
}},
{{
name: "INPUT1"
data_type: {}
dims: [ {} ]
}}
]
output [
{{
name: "OUTPUT0"
data_type: {}
dims: [ {} ]
}},
{{
name: "OUTPUT1"
data_type: {}
dims: [ {} ]
}}
]
'''.format(model_name, max_batch, version_policy_str,
np_to_model_dtype(input_dtype), tu.shape_to_dims_str(input_shape),
np_to_model_dtype(input_dtype), tu.shape_to_dims_str(input_shape),
np_to_model_dtype(output0_dtype),
tu.shape_to_dims_str(output0_shape),
np_to_model_dtype(output1_dtype),
tu.shape_to_dims_str(output1_shape))
try:
os.makedirs(config_dir)
except OSError as ex:
pass # ignore existing dir
with open(config_dir + "/config.pbtxt", "w") as cfile:
cfile.write(config)
