def test_default_config_single_model(self):
"""
Test Default Single Model:
num_PAC = log2(DEFAULT_RUN_CONFIG_MAX_CONCURRENCY) + 1
num_MC = ( DEFAULT_RUN_CONFIG_MAX_INSTANCE_COUNT
x log2(DEFAULT_RUN_CONFIG_MAX_MODEL_BATCH_SIZE)
) + 1
total = (num_PAC * num_MC) will be generated by the auto-search
"""
# yapf: disable
yaml_content = convert_to_bytes("""
profile_models:
- my-model
""")
# yapf: enable
expected_pa_configs = len(
utils.generate_log2_list(DEFAULT_RUN_CONFIG_MAX_CONCURRENCY))
expected_model_configs = DEFAULT_RUN_CONFIG_MAX_INSTANCE_COUNT \
* len(utils.generate_log2_list(DEFAULT_RUN_CONFIG_MAX_MODEL_BATCH_SIZE)) \
+ 1
expected_num_of_configs = expected_pa_configs * expected_model_configs
self._run_and_test_run_config_generator(
yaml_content, expected_config_count=expected_num_of_configs)
def test_default(self):
"""
Test Default:
- No CLI options specified
Default (1) value will be used for batch size
and log2(DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)+1 configs
will be generated by the auto-search
"""
# yapf: disable
yaml_content = convert_to_bytes("""
profile_models:
- my-model
""")
# yapf: enable
concurrencies = utils.generate_log2_list(
DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)
expected_configs = [
construct_perf_analyzer_config(concurrency=c)
for c in concurrencies
]
self._run_and_test_perf_analyzer_config_generator(
yaml_content, expected_configs)
def test_c_api(self):
"""
Test C_API:
- Launch mode is C_API
Default (1) values will be used for batch size/concurrency
and only one config will be generated
"""
# yapf: disable
yaml_content = convert_to_bytes("""
profile_models:
- my-model
""")
# yapf: enable
concurrencies = utils.generate_log2_list(
DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)
expected_configs = [
construct_perf_analyzer_config(concurrency=c, launch_mode='c_api')
for c in concurrencies
]
self._run_and_test_perf_analyzer_config_generator(
yaml_content, expected_configs, '--triton-launch-mode=c_api')
def test_perf_analyzer_flags(self):
"""
Test Perf Analyzer Flags:
- No CLI options specified
- Percentile (PA flag) set in model's YAML
Default (1) value will be used for batch size
and log2(DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)+1 configs
will be generated by the auto-search
"""
# yapf: disable
yaml_content = convert_to_bytes("""
profile_models:
- my-model:
perf_analyzer_flags:
percentile: 96
""")
# yapf: enable
concurrencies = utils.generate_log2_list(
DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)
expected_configs = [
construct_perf_analyzer_config(
concurrency=c, perf_analyzer_flags={'percentile': '96'})
for c in concurrencies
]
self._run_and_test_perf_analyzer_config_generator(
yaml_content, expected_configs)
def test_max_concurrency(self):
"""
Test Max Concurrency:
- Change max concurrency to non-default value
Max Concurrency: 16
Default (1) value will be used for batch size
and 5 configs (log2(16)+1) will be generated
"""
# yapf: disable
yaml_content = convert_to_bytes("""
profile_models:
- my-model
""")
# yapf: enable
concurrencies = utils.generate_log2_list(16)
expected_configs = [
construct_perf_analyzer_config(concurrency=c)
for c in concurrencies
]
pa_cli_args = ['--run-config-search-max-concurrency', '16']
self._run_and_test_perf_analyzer_config_generator(
yaml_content, expected_configs, pa_cli_args)
def test_batch_size_search_enabled(self):
"""
Test Batch Size Search Enabled:
- Schmoo batch sizes
- Run Config Search enabled
Batch sizes: 1,2,4
Concurrency: log2(DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)+1
"""
# yapf: disable
yaml_content = convert_to_bytes("""
profile_models:
- my-model
""")
# yapf: enable
batch_sizes = [1, 2, 4]
concurrencies = utils.generate_log2_list(
DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)
expected_configs = [
construct_perf_analyzer_config(batch_size=b, concurrency=c)
for b in batch_sizes for c in concurrencies
]
pa_cli_args = ['-b 1,2,4']
self._run_and_test_perf_analyzer_config_generator(
yaml_content, expected_configs, pa_cli_args)
def test_http(self):
"""
Test HTTP:
- Client protocol is HTTP
Default (1) values will be used for batch size/concurrency
and only one config will be generated
"""
# yapf: disable
yaml_content = convert_to_bytes("""
profile_models:
- my-model
""")
# yapf: enable
concurrencies = utils.generate_log2_list(
DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)
expected_configs = [
construct_perf_analyzer_config(concurrency=c,
client_protocol='http')
for c in concurrencies
]
self._run_and_test_perf_analyzer_config_generator(
yaml_content, expected_configs, '--client-protocol=http')
def test_perf_analyzer_config_ssl_options(self):
"""
Test Perf Analyzer SSL options:
- No CLI options specified
"""
# yapf: disable
yaml_content = convert_to_bytes("""
profile_models:
- my-model:
perf_analyzer_flags:
ssl-grpc-root-certifications-file: a
ssl-grpc-private-key-file: b
ssl-grpc-certificate-chain-file: c
ssl-https-verify-peer: 1
ssl-https-verify-host: 2
ssl-https-ca-certificates-file: d
ssl-https-client-certificate-type: e
ssl-https-client-certificate-file: f
ssl-https-private-key-type: g
ssl-https-private-key-file: h
""")
# yapf: enable
concurrencies = utils.generate_log2_list(
DEFAULT_RUN_CONFIG_MAX_CONCURRENCY)
expected_configs = [
construct_perf_analyzer_config(
concurrency=c,
perf_analyzer_flags={
'ssl-grpc-root-certifications-file': 'a',
'ssl-grpc-private-key-file': 'b',
'ssl-grpc-certificate-chain-file': 'c',
'ssl-https-verify-peer': '1',
'ssl-https-verify-host': '2',
'ssl-https-ca-certificates-file': 'd',
'ssl-https-client-certificate-type': 'e',
'ssl-https-client-certificate-file': 'f',
'ssl-https-private-key-type': 'g',
'ssl-https-private-key-file': 'h',
}) for c in concurrencies
]
self._run_and_test_perf_analyzer_config_generator(
yaml_content, expected_configs)