def create_from_file(model_path):
"""
Constructs a ModelConfig from the pbtxt at file
Parameters
-------
model_path : str
The full path to this model directory
Returns
-------
ModelConfig
"""
if not os.path.exists(model_path):
raise TritonModelAnalyzerException(
'Model path specified does not exist.')
if os.path.isfile(model_path):
raise TritonModelAnalyzerException(
'Model output path must be a directory.')
with open(os.path.join(model_path, "config.pbtxt"), 'r+') as f:
config_str = f.read()
protobuf_message = text_format.Parse(config_str,
model_config_pb2.ModelConfig())
return ModelConfig(protobuf_message)
def mock_top_n_results(self, model_name=None, n=-1):
return [
ModelResult(
None,
ModelConfig(
json_format.ParseDict({'name': 'config1'},
model_config_pb2.ModelConfig())),
None),
ModelResult(
None,
ModelConfig(
json_format.ParseDict({'name': 'config3'},
model_config_pb2.ModelConfig())),
None),
ModelResult(
None,
ModelConfig(
json_format.ParseDict({'name': 'config4'},
model_config_pb2.ModelConfig())),
None)
]
def set_config(self, config):
"""
Set the model config from a dictionary.
Parameters
----------
config : dict
The new dictionary containing the model config.
"""
self._model_config = json_format.ParseDict(
config, model_config_pb2.ModelConfig())
def create_from_dictionary(model_dict):
"""
Constructs a ModelConfig from a Python dictionary
Parameters
-------
model_dict : dict
A dictionary containing the model configuration.
Returns
-------
ModelConfig
"""
protobuf_message = json_format.ParseDict(
model_dict, model_config_pb2.ModelConfig())
return ModelConfig(protobuf_message)
def create_from_file(model_path):
"""
Constructs a ModelConfig from the pbtxt at file
Parameters
-------
model_path : str
The full path to this model directory
Returns
-------
ModelConfig
"""
if not os.path.exists(model_path):
raise TritonModelAnalyzerException(
f'Model path "{model_path}" specified does not exist.')
if os.path.isfile(model_path):
raise TritonModelAnalyzerException(
f'Model output path "{model_path}" must be a directory.')
model_config_path = os.path.join(model_path, "config.pbtxt")
if not os.path.isfile(model_config_path):
raise TritonModelAnalyzerException(
f'Path "{model_config_path}" does not exist.'
' Make sure that you have specified the correct model'
' repository and model name(s).')
with open(model_config_path, 'r+') as f:
config_str = f.read()
protobuf_message = text_format.Parse(config_str,
model_config_pb2.ModelConfig())
return ModelConfig(protobuf_message)
verbose=False)
model_config = triton_client.get_model_config(model_name)
else:
triton_client = grpcclient.InferenceServerClient(url=pair[0],
verbose=False)
model_config = triton_client.get_model_config(model_name)
nonmatch = list()
expected_files = [
f for f in os.listdir(FLAGS.expected_dir)
if (os.path.isfile(os.path.join(FLAGS.expected_dir, f)) and (
f.startswith("expected")))
]
for efile in expected_files:
with open(os.path.join(FLAGS.expected_dir, efile)) as f:
config = text_format.Parse(f.read(), mc.ModelConfig())
if pair[1] == "http":
config_json = json.loads(
json_format.MessageToJson(
config, preserving_proto_field_name=True))
if config_json == model_config:
sys.exit(0)
else:
if config == model_config.config:
sys.exit(0)
nonmatch.append(config)
print("Model config doesn't match any expected output:")
print("Model config:")
def generate_ma_result(json_file_path, result_pbtxt_path, ma_config_path):
from google.protobuf import text_format, json_format
from tritonclient.grpc import model_config_pb2
with open(json_file_path) as json_file:
olive_result = json.load(json_file)
results = olive_result.get("all_tuning_results")
best_test_name = olive_result.get("best_test_name")
for result in results:
if result.get("test_name") == best_test_name:
execution_provider = result.get("execution_provider")
env_vars = result.get("env_vars")
session_options = result.get("session_options")
break
optimization_config = None
sess_opt_parameters = None
if best_test_name == "pretuning":
optimization_config = {"graph": {"level": 1}}
else:
intra_op_thread_count = session_options.get("intra_op_num_threads")
inter_op_thread_count = session_options.get("inter_op_num_threads")
execution_mode = session_options.get("execution_mode")
graph_optimization_level = session_options.get("graph_optimization_level")
if graph_optimization_level in ["0", "1"]:
opt_level = -1
else:
opt_level = 1
if execution_provider == "TensorrtExecutionProvider":
tensorrt_accelerator = {"name": "tensorrt"}
if env_vars.get("ORT_TENSORRT_FP16_ENABLE") == "1":
tensorrt_accelerator["parameters"] = {"precision_mode": "FP16"}
optimization_config = {
"executionAccelerators": {"gpuExecutionAccelerator": [tensorrt_accelerator]},
"graph": {"level": opt_level}
}
elif execution_provider == "OpenVINOExecutionProvider":
optimization_config = {
"executionAccelerators": {"cpuExecutionAccelerator": [{"name": "openvino"}]},
"graph": {"level": opt_level}}
else:
optimization_config = {"graph": {"level": opt_level}}
sess_opt_parameters = {}
if intra_op_thread_count != "None":
sess_opt_parameters["intra_op_thread_count"] = {"stringValue": intra_op_thread_count}
if inter_op_thread_count != "None":
sess_opt_parameters["inter_op_thread_count"] = {"stringValue": inter_op_thread_count}
if execution_mode:
execution_mode_flag = "0" if execution_mode == "ExecutionMode.ORT_SEQUENTIAL" else "1"
sess_opt_parameters["execution_mode"] = {"stringValue": execution_mode_flag}
with open(ma_config_path, 'r+') as f:
config_str = f.read()
protobuf_message = text_format.Parse(config_str, model_config_pb2.ModelConfig())
model_dict = json_format.MessageToDict(protobuf_message)
model_dict.update({"optimization": optimization_config})
if sess_opt_parameters:
model_dict.update({"parameters": sess_opt_parameters})
protobuf_message = json_format.ParseDict(model_dict, model_config_pb2.ModelConfig())
model_config_bytes = text_format.MessageToBytes(protobuf_message)
with open(result_pbtxt_path, "wb") as f:
f.write(model_config_bytes)
def test_get_common_row_items_with_backend_parameters(self):
"""
This tests that a metrics model inference table row can be created with
backend parameters included. Each backend parameter gets its own column.
The column name is the backend parameter key (prepended with a prefix
to avoid potentially overlapping with an existing column). The column
value is the backend parameter value.
Here is an example table:
Models (Inference):
Model Model Config Path backend_parameter/add_sub_key_1 backend_parameter/add_sub_key_2
add_sub add_sub_config_2 add_sub_value_1 add_sub_value_2
add_sub add_sub_config_0 add_sub_value_1 add_sub_value_2
add_sub add_sub_config_1 add_sub_value_1 add_sub_value_2
Each row of the metrics model inference table corresponds to one model
config variant.
It is possible for a user to run the analyze command with multiple
models config variants from different models with potentially different
backend parameters. This test includes backend parameters from two
separate models, showing that for one particular row (for a 'model A'
config variant), it only populates the backend parameter cells for
'model A', and the backend parameter cells for 'model B' are empty
(None).
Here is an example table with backend parameters from different models:
Models (Inference):
Model Model Config Path backend_parameter/add_sub_key_1 backend_parameter/add_sub_key_2 backend_parameter/add_sub_2_key_1 backend_parameter/add_sub_2_key_2
add_sub add_sub_config_2 add_sub_value_1 add_sub_value_2 None None
add_sub add_sub_config_0 add_sub_value_1 add_sub_value_2 None None
add_sub add_sub_config_1 add_sub_value_1 add_sub_value_2 None None
add_sub_2 add_sub_2_config_2 None None add_sub_2_value_1 add_sub_2_value_2
add_sub_2 add_sub_2_config_1 None None add_sub_2_value_1 add_sub_2_value_2
add_sub_2 add_sub_2_config_0 None None add_sub_2_value_1 add_sub_2_value_2
"""
args = ['model-analyzer', 'analyze', '-f', 'config.yml']
yaml_content = convert_to_bytes("""
analysis_models: analysis_models
inference_output_fields: model_name,batch_size,backend_parameter/model_1_key_1,backend_parameter/model_1_key_2,backend_parameter/model_2_key_1
""")
config = self._evaluate_config(args, yaml_content)
state_manager = AnalyzerStateManager(config=config, server=None)
result_manager = ResultManager(config=config,
state_manager=state_manager)
model_config_str = """
parameters: {
key: "model_1_key_1"
value: {
string_value:"model_1_value_1"
}
}
parameters: {
key:"model_1_key_2"
value: {
string_value:"model_1_value_2"
}
}
"""
backend_parameters = text_format.Parse(
model_config_str, model_config_pb2.ModelConfig()).parameters
row = result_manager._get_common_row_items(
fields=[
'model_name', 'batch_size', 'backend_parameter/model_1_key_1',
'backend_parameter/model_1_key_2',
'backend_parameter/model_2_key_1'
],
batch_size='batch_size',
concurrency=None,
satisfies=None,
model_name='model_name',
model_config_path=None,
dynamic_batching=None,
instance_group=None,
backend_parameters=backend_parameters)
self.assertTrue(row == [
'model_name', 'batch_size', 'model_1_value_1', 'model_1_value_2',
None
])
