def create_from_file(model_path): """ Constructs a ModelConfig from the pbtxt at file Parameters ------- model_path : str The full path to this model directory Returns ------- ModelConfig """ if not os.path.exists(model_path): raise TritonModelAnalyzerException( 'Model path specified does not exist.') if os.path.isfile(model_path): raise TritonModelAnalyzerException( 'Model output path must be a directory.') with open(os.path.join(model_path, "config.pbtxt"), 'r+') as f: config_str = f.read() protobuf_message = text_format.Parse(config_str, model_config_pb2.ModelConfig()) return ModelConfig(protobuf_message)
def mock_top_n_results(self, model_name=None, n=-1): return [ ModelResult( None, ModelConfig( json_format.ParseDict({'name': 'config1'}, model_config_pb2.ModelConfig())), None), ModelResult( None, ModelConfig( json_format.ParseDict({'name': 'config3'}, model_config_pb2.ModelConfig())), None), ModelResult( None, ModelConfig( json_format.ParseDict({'name': 'config4'}, model_config_pb2.ModelConfig())), None) ]
def set_config(self, config): """ Set the model config from a dictionary. Parameters ---------- config : dict The new dictionary containing the model config. """ self._model_config = json_format.ParseDict( config, model_config_pb2.ModelConfig())
def create_from_dictionary(model_dict): """ Constructs a ModelConfig from a Python dictionary Parameters ------- model_dict : dict A dictionary containing the model configuration. Returns ------- ModelConfig """ protobuf_message = json_format.ParseDict( model_dict, model_config_pb2.ModelConfig()) return ModelConfig(protobuf_message)
def create_from_file(model_path): """ Constructs a ModelConfig from the pbtxt at file Parameters ------- model_path : str The full path to this model directory Returns ------- ModelConfig """ if not os.path.exists(model_path): raise TritonModelAnalyzerException( f'Model path "{model_path}" specified does not exist.') if os.path.isfile(model_path): raise TritonModelAnalyzerException( f'Model output path "{model_path}" must be a directory.') model_config_path = os.path.join(model_path, "config.pbtxt") if not os.path.isfile(model_config_path): raise TritonModelAnalyzerException( f'Path "{model_config_path}" does not exist.' ' Make sure that you have specified the correct model' ' repository and model name(s).') with open(model_config_path, 'r+') as f: config_str = f.read() protobuf_message = text_format.Parse(config_str, model_config_pb2.ModelConfig()) return ModelConfig(protobuf_message)
verbose=False) model_config = triton_client.get_model_config(model_name) else: triton_client = grpcclient.InferenceServerClient(url=pair[0], verbose=False) model_config = triton_client.get_model_config(model_name) nonmatch = list() expected_files = [ f for f in os.listdir(FLAGS.expected_dir) if (os.path.isfile(os.path.join(FLAGS.expected_dir, f)) and ( f.startswith("expected"))) ] for efile in expected_files: with open(os.path.join(FLAGS.expected_dir, efile)) as f: config = text_format.Parse(f.read(), mc.ModelConfig()) if pair[1] == "http": config_json = json.loads( json_format.MessageToJson( config, preserving_proto_field_name=True)) if config_json == model_config: sys.exit(0) else: if config == model_config.config: sys.exit(0) nonmatch.append(config) print("Model config doesn't match any expected output:") print("Model config:")
def generate_ma_result(json_file_path, result_pbtxt_path, ma_config_path): from google.protobuf import text_format, json_format from tritonclient.grpc import model_config_pb2 with open(json_file_path) as json_file: olive_result = json.load(json_file) results = olive_result.get("all_tuning_results") best_test_name = olive_result.get("best_test_name") for result in results: if result.get("test_name") == best_test_name: execution_provider = result.get("execution_provider") env_vars = result.get("env_vars") session_options = result.get("session_options") break optimization_config = None sess_opt_parameters = None if best_test_name == "pretuning": optimization_config = {"graph": {"level": 1}} else: intra_op_thread_count = session_options.get("intra_op_num_threads") inter_op_thread_count = session_options.get("inter_op_num_threads") execution_mode = session_options.get("execution_mode") graph_optimization_level = session_options.get("graph_optimization_level") if graph_optimization_level in ["0", "1"]: opt_level = -1 else: opt_level = 1 if execution_provider == "TensorrtExecutionProvider": tensorrt_accelerator = {"name": "tensorrt"} if env_vars.get("ORT_TENSORRT_FP16_ENABLE") == "1": tensorrt_accelerator["parameters"] = {"precision_mode": "FP16"} optimization_config = { "executionAccelerators": {"gpuExecutionAccelerator": [tensorrt_accelerator]}, "graph": {"level": opt_level} } elif execution_provider == "OpenVINOExecutionProvider": optimization_config = { "executionAccelerators": {"cpuExecutionAccelerator": [{"name": "openvino"}]}, "graph": {"level": opt_level}} else: optimization_config = {"graph": {"level": opt_level}} sess_opt_parameters = {} if intra_op_thread_count != "None": sess_opt_parameters["intra_op_thread_count"] = {"stringValue": intra_op_thread_count} if inter_op_thread_count != "None": sess_opt_parameters["inter_op_thread_count"] = {"stringValue": inter_op_thread_count} if execution_mode: execution_mode_flag = "0" if execution_mode == "ExecutionMode.ORT_SEQUENTIAL" else "1" sess_opt_parameters["execution_mode"] = {"stringValue": execution_mode_flag} with open(ma_config_path, 'r+') as f: config_str = f.read() protobuf_message = text_format.Parse(config_str, model_config_pb2.ModelConfig()) model_dict = json_format.MessageToDict(protobuf_message) model_dict.update({"optimization": optimization_config}) if sess_opt_parameters: model_dict.update({"parameters": sess_opt_parameters}) protobuf_message = json_format.ParseDict(model_dict, model_config_pb2.ModelConfig()) model_config_bytes = text_format.MessageToBytes(protobuf_message) with open(result_pbtxt_path, "wb") as f: f.write(model_config_bytes)
def test_get_common_row_items_with_backend_parameters(self): """ This tests that a metrics model inference table row can be created with backend parameters included. Each backend parameter gets its own column. The column name is the backend parameter key (prepended with a prefix to avoid potentially overlapping with an existing column). The column value is the backend parameter value. Here is an example table: Models (Inference): Model Model Config Path backend_parameter/add_sub_key_1 backend_parameter/add_sub_key_2 add_sub add_sub_config_2 add_sub_value_1 add_sub_value_2 add_sub add_sub_config_0 add_sub_value_1 add_sub_value_2 add_sub add_sub_config_1 add_sub_value_1 add_sub_value_2 Each row of the metrics model inference table corresponds to one model config variant. It is possible for a user to run the analyze command with multiple models config variants from different models with potentially different backend parameters. This test includes backend parameters from two separate models, showing that for one particular row (for a 'model A' config variant), it only populates the backend parameter cells for 'model A', and the backend parameter cells for 'model B' are empty (None). Here is an example table with backend parameters from different models: Models (Inference): Model Model Config Path backend_parameter/add_sub_key_1 backend_parameter/add_sub_key_2 backend_parameter/add_sub_2_key_1 backend_parameter/add_sub_2_key_2 add_sub add_sub_config_2 add_sub_value_1 add_sub_value_2 None None add_sub add_sub_config_0 add_sub_value_1 add_sub_value_2 None None add_sub add_sub_config_1 add_sub_value_1 add_sub_value_2 None None add_sub_2 add_sub_2_config_2 None None add_sub_2_value_1 add_sub_2_value_2 add_sub_2 add_sub_2_config_1 None None add_sub_2_value_1 add_sub_2_value_2 add_sub_2 add_sub_2_config_0 None None add_sub_2_value_1 add_sub_2_value_2 """ args = ['model-analyzer', 'analyze', '-f', 'config.yml'] yaml_content = convert_to_bytes(""" analysis_models: analysis_models inference_output_fields: model_name,batch_size,backend_parameter/model_1_key_1,backend_parameter/model_1_key_2,backend_parameter/model_2_key_1 """) config = self._evaluate_config(args, yaml_content) state_manager = AnalyzerStateManager(config=config, server=None) result_manager = ResultManager(config=config, state_manager=state_manager) model_config_str = """ parameters: { key: "model_1_key_1" value: { string_value:"model_1_value_1" } } parameters: { key:"model_1_key_2" value: { string_value:"model_1_value_2" } } """ backend_parameters = text_format.Parse( model_config_str, model_config_pb2.ModelConfig()).parameters row = result_manager._get_common_row_items( fields=[ 'model_name', 'batch_size', 'backend_parameter/model_1_key_1', 'backend_parameter/model_1_key_2', 'backend_parameter/model_2_key_1' ], batch_size='batch_size', concurrency=None, satisfies=None, model_name='model_name', model_config_path=None, dynamic_batching=None, instance_group=None, backend_parameters=backend_parameters) self.assertTrue(row == [ 'model_name', 'batch_size', 'model_1_value_1', 'model_1_value_2', None ])