def _generate_run_configs(self):
analyzer_config = self._analyzer_config
model_repository = analyzer_config['model_repository']
model = self._model
model_name_index = 0
model_config_parameters = model.model_config_parameters()
# Generate all the sweeps for a given parameter
models_sweeps = \
self._generate_model_config_combinations(
model_config_parameters)
for model_sweep in models_sweeps:
model_config = ModelConfig.create_from_file(
f'{model_repository}/{model.model_name()}')
model_config_dict = model_config.get_config()
for key, value in model_sweep.items():
model_config_dict[key] = value
model_config = ModelConfig.create_from_dictionary(
model_config_dict)
# Temporary model name to be used for profiling. We
# can't use the same name for different configurations.
# The new model name is the original model suffixed with
# _i<config_index>. Where the config index is the index
# of the model config alternative.
model_tmp_name = f'{model.model_name()}_i{model_name_index}'
model_config.set_field('name', model_tmp_name)
perf_configs = self._generate_perf_config_for_model(
model_tmp_name, model)
# Add the new run config.
self._run_configs.append(
RunConfig(model.model_name(), model_config, perf_configs))
model_name_index += 1
def generate_run_config_for_model_sweep(self, model, model_sweep):
"""
Parameters
----------
model : ConfigModel
The model for which a run config is being generated
model_sweep: dict
Model config parameters
"""
analyzer_config = self._analyzer_config
model_repository = analyzer_config['model_repository']
num_retries = analyzer_config['max_retries']
if analyzer_config['triton_launch_mode'] != 'remote':
model_config = ModelConfig.create_from_file(
f'{model_repository}/{model.model_name()}')
if model_sweep is not None:
model_config_dict = model_config.get_config()
for key, value in model_sweep.items():
if value is not None:
model_config_dict[key] = value
model_config = ModelConfig.create_from_dictionary(
model_config_dict)
model_name_index = self._model_name_index
model_config_dict = model_config.get_config()
try:
model_name_index = self._model_configs.index(model_config_dict)
except ValueError:
self._model_configs.append(model_config_dict)
self._model_name_index += 1
# Temporary model name to be used for profiling. We
# can't use the same name for different configurations.
# The new model name is the original model suffixed with
# _i<config_index>. Where the config index is the index
# of the model config alternative.
model_tmp_name = f'{model.model_name()}_i{model_name_index}'
model_config.set_field('name', model_tmp_name)
perf_configs = self._generate_perf_config_for_model(
model_tmp_name, model)
for perf_config in perf_configs:
self._run_configs.append(
RunConfig(model.model_name(), model_config, perf_config))
else:
model_config = ModelConfig.create_from_triton_api(
self._client, model.model_name(), num_retries)
perf_configs = self._generate_perf_config_for_model(
model.model_name(), model)
for perf_config in perf_configs:
# Add the new run config.
self._run_configs.append(
RunConfig(model.model_name(), model_config, perf_config))
def test_create_from_dict(self):
model_config = ModelConfig.create_from_dictionary(self._model_config)
self.assertTrue(model_config.get_config() == self._model_config)
new_config = {'instance_group': [{'count': 2, 'kind': 'KIND_CPU'}]}
model_config.set_config(new_config)
self.assertTrue(model_config.get_config() == new_config)
def test_create_from_file(self):
test_protobuf = self._model_config_protobuf
mock_model_config = MockModelConfig(test_protobuf)
mock_model_config.start()
model_config = ModelConfig.create_from_file('/path/to/model_config')
self.assertTrue(model_config.get_config() == self._model_config)
mock_model_config.stop()
def test_write_config_to_file_with_relative_path(self, mock_os_symlink,
*args):
"""
Tests that the call to os.symlink() within write_config_to_file() uses
a valid relative path when user uses a relative path with the
`--output-model-repository-path` option
"""
model_config = ModelConfig.create_from_dictionary(self._model_config)
model_path = './output_model_repository/model_config_1'
src_model_path = '/tmp/src_model_repository/model'
last_model_path = './output_model_repository/model_config_0'
mock_model_config = MockModelConfig()
mock_model_config.start()
model_config.write_config_to_file(model_path, src_model_path,
last_model_path)
mock_model_config.stop()
mock_os_symlink.assert_any_call(
'../model_config_0/1',
'./output_model_repository/model_config_1/1')
mock_os_symlink.assert_any_call(
'../model_config_0/output0_labels.txt',
'./output_model_repository/model_config_1/output0_labels.txt')
def from_dict(cls, results_dict):
"""
Populate the Results class based on the dictionary value
(stored in the checkpoint)
The checkpoint format is:
{model_name: { [model_config: (key, {measurements} ) ] } }
---results_dict-------------------------------------------
---model_dict------------------------------
---model_config_tuple_list-------------
-key, measurement_dict-
"""
results = Results()
for model_name, model_dict in results_dict['_results'].items():
for model_config_name, model_config_tuple_list in model_dict.items(
):
model_config = ModelConfig.from_dict(
model_config_tuple_list[Results.MODEL_CONFIG_INDEX])
for key, measurement_dict in model_config_tuple_list[
Results.MEASUREMENTS_INDEX].items():
measurement = Measurement.from_dict(measurement_dict)
results._add_measurement(model_name, model_config,
model_config_name, key,
measurement)
return results
def _create_model_run_config(self, model_name, model_config_name,
pa_config_name):
model_config_dict = {'name': model_config_name}
self._model_config = ModelConfig.create_from_dictionary(
model_config_dict)
perf_config = PerfAnalyzerConfig()
perf_config.update_config({'model-name': pa_config_name})
return ModelRunConfig(model_name, self._model_config, perf_config)
def _make_remote_model_config(self):
if not self._reload_model_disable:
self._client.load_model(self._base_model_name)
model_config = ModelConfig.create_from_triton_api(
self._client, self._base_model_name, self._num_retries)
model_config.set_cpu_only(self._cpu_only)
if not self._reload_model_disable:
self._client.unload_model(self._base_model_name)
return model_config
def mock_top_n_results(self, model_name=None, n=-1):
return [
ModelResult(
None,
ModelConfig(
json_format.ParseDict({'name': 'config1'},
model_config_pb2.ModelConfig())),
None),
ModelResult(
None,
ModelConfig(
json_format.ParseDict({'name': 'config3'},
model_config_pb2.ModelConfig())),
None),
ModelResult(
None,
ModelConfig(
json_format.ParseDict({'name': 'config4'},
model_config_pb2.ModelConfig())),
None)
]
def test_write_config_file(self):
model_config = ModelConfig.create_from_dictionary(self._model_config)
model_output_path = os.path.abspath('./model_config')
mock_model_config = MockModelConfig()
mock_model_config.start()
# Write the model config to output
with patch('model_analyzer.triton.model.model_config.open',
mock_open()) as mocked_file:
with patch('model_analyzer.triton.model.model_config.copy_tree',
MagicMock()):
model_config.write_config_to_file(model_output_path,
'/mock/path', None)
content = mocked_file().write.call_args.args[0]
mock_model_config.stop()
mock_model_config = MockModelConfig(content)
mock_model_config.start()
model_config_from_file = \
ModelConfig.create_from_file(model_output_path)
self.assertTrue(
model_config_from_file.get_config() == self._model_config)
mock_model_config.stop()
# output path doesn't exist
with patch('model_analyzer.triton.model.model_config.os.path.exists',
MagicMock(return_value=False)):
with self.assertRaises(TritonModelAnalyzerException):
ModelConfig.create_from_file(model_output_path)
# output path is a file
with patch('model_analyzer.triton.model.model_config.os.path.isfile',
MagicMock(return_value=True)):
with self.assertRaises(TritonModelAnalyzerException):
ModelConfig.create_from_file(model_output_path)
def _make_direct_mode_model_config(self, param_combo):
"""
Given a base model config and a combination of parameters to change,
apply the changes on top of the base and return the new model config
"""
model_config_dict = self._get_base_model_config_dict()
if param_combo is not None:
for key, value in param_combo.items():
if value is not None:
model_config_dict[key] = value
model_config_dict['name'] = self._get_model_variant_name(param_combo)
model_config = ModelConfig.create_from_dictionary(model_config_dict)
return model_config
def test_build_summary_table(self):
mock_model_config = MockModelConfig()
mock_model_config.start()
objective_spec = {'perf_throughput': 10}
self.result_comparator = ResultComparator(
metric_objectives=objective_spec)
avg_gpu_metrics = {0: {'gpu_used_memory': 6000, 'gpu_utilization': 60}}
for i in range(10, 0, -1):
avg_non_gpu_metrics = {
'perf_throughput': 100 + 10 * i,
'perf_latency': 4000
}
self.model_config['name'] = f'model_{i}'
model_config = ModelConfig.create_from_dictionary(
self.model_config)
self.report_manager.add_result(
report_key='test_report',
result=construct_result(
avg_gpu_metric_values=avg_gpu_metrics,
avg_non_gpu_metric_values=avg_non_gpu_metrics,
comparator=self.result_comparator,
model_config=model_config))
summary_table, summary_sentence = \
self.report_manager._build_summary_table(
report_key='test_report',
num_measurements=10,
gpu_name='TITAN RTX')
expected_summary_sentence = (
"In 10 measurement(s), 1/GPU model instance(s)"
" with max dynamic batch size of [4 8] on"
" platform tensorflow_graphdef delivers maximum"
" throughput under the given constraints on GPU(s) TITAN RTX.")
self.assertEqual(expected_summary_sentence, summary_sentence)
# Get throughput index and make sure results are sorted
throughput_index = summary_table.headers().index(
'Throughput (infer/sec)')
model_name_index = summary_table.headers().index('Model Config Name')
for i in range(9):
current_row = summary_table.get_row_by_index(i)
next_row = summary_table.get_row_by_index(i + 1)
self.assertEqual(current_row[model_name_index], f'model_{10-i}')
self.assertGreaterEqual(current_row[throughput_index],
next_row[throughput_index])
def _add_result_measurement(self,
model_config_name,
model_name,
avg_gpu_metrics,
avg_non_gpu_metrics,
result_comparator,
cpu_only=False):
config_pb = self.model_config.copy()
config_pb["name"] = model_config_name
model_config = ModelConfig.create_from_dictionary(config_pb)
model_config._cpu_only = cpu_only
measurement = construct_measurement(model_name, avg_gpu_metrics,
avg_non_gpu_metrics,
result_comparator)
run_config = ModelRunConfig(model_name, model_config,
measurement.perf_config())
self.result_manager.add_measurement(run_config, measurement)
def _get_base_model_config_dict(self):
config = ModelConfig.create_from_file(
f'{self._model_repository}/{self._base_model_name}')
return config.get_config()