def test_pytorch_sharding_matmul():
"""Test pytorch-sharding-matmul benchmark."""
context = BenchmarkRegistry.create_benchmark_context(
'sharding-matmul',
platform=Platform.CUDA,
parameters='--run_count 2 --num_steps 20',
framework=Framework.PYTORCH)
assert (BenchmarkRegistry.is_benchmark_context_valid(context))
port = network.get_free_port()
assert (port)
utils.setup_simulated_ddp_distributed_env(1, 0, port)
benchmark = BenchmarkRegistry.launch_benchmark(context)
# Check basic information.
assert (benchmark)
assert (isinstance(benchmark, ShardingMatmul))
assert (benchmark.name == 'pytorch-sharding-matmul')
assert (benchmark.type == BenchmarkType.MICRO)
# Check predefined parameters of sharding-matmul benchmark.
assert (benchmark._args.mode == [
ShardingMode.ALLREDUCE, ShardingMode.ALLGATHER
])
# Check parameters specified in BenchmarkContext.
assert (benchmark._args.run_count == 2)
assert (benchmark._args.num_steps == 20)
# Check results and metrics.
assert (benchmark.run_count == 2)
assert (benchmark.return_code == ReturnCode.SUCCESS)
for metric in ['allreduce_time', 'allgather_time']:
assert (len(benchmark.raw_data[metric]) == benchmark.run_count)
assert (len(
benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
assert (len(benchmark.result[metric]) == benchmark.run_count)
utils.clean_simulated_ddp_distributed_env()
def test_cublas_functions():
"""Test cublas-function benchmark."""
# Test for default configuration
context = BenchmarkRegistry.create_benchmark_context(
'cublas-function',
platform=Platform.CUDA,
parameters='--num_warmup 10 --num_steps 10 --num_in_step 100')
assert (BenchmarkRegistry.is_benchmark_context_valid(context))
benchmark = BenchmarkRegistry.launch_benchmark(context)
# Check basic information.
assert (benchmark)
assert (benchmark.name == 'cublas-function')
assert (benchmark.type == BenchmarkType.MICRO)
# Check parameters specified in BenchmarkContext.
assert (benchmark._args.num_warmup == 10)
assert (benchmark._args.num_steps == 10)
assert (benchmark._args.num_in_step == 100)
# Check results and metrics.
assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS)
assert ('raw_output_0' in benchmark.raw_data)
assert (len(benchmark.raw_data['raw_output_0']) == 1)
assert (isinstance(benchmark.raw_data['raw_output_0'][0], str))
assert (19 <= len(benchmark.result))
for metric in list(benchmark.result.keys()):
assert (len(benchmark.result[metric]) == 1)
assert (isinstance(benchmark.result[metric][0], numbers.Number))
if metric != 'return_code':
assert (len(
benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
# Test for custom configuration
custom_config_str = '{"name":"cublasCgemm","m":512,"n":512,"k":32,"transa":1,"transb":0}'
context = BenchmarkRegistry.create_benchmark_context(
'cublas-function',
platform=Platform.CUDA,
parameters=
'--num_warmup 10 --num_steps 10 --num_in_step 100 --config_json_str ' +
custom_config_str)
assert (BenchmarkRegistry.is_benchmark_context_valid(context))
benchmark = BenchmarkRegistry.launch_benchmark(context)
# Check basic information.
assert (benchmark)
assert (benchmark.name == 'cublas-function')
assert (benchmark.type == BenchmarkType.MICRO)
# Check parameters specified in BenchmarkContext.
assert (benchmark._args.num_warmup == 10)
assert (benchmark._args.num_steps == 10)
assert (benchmark._args.num_in_step == 100)
# Check results and metrics.
assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS)
assert ('raw_output_0' in benchmark.raw_data)
assert (len(benchmark.raw_data['raw_output_0']) == 1)
assert (isinstance(benchmark.raw_data['raw_output_0'][0], str))
assert (1 + benchmark.default_metric_count == len(benchmark.result))
for metric in list(benchmark.result.keys()):
assert (len(benchmark.result[metric]) == 1)
assert (isinstance(benchmark.result[metric][0], numbers.Number))
if metric != 'return_code':
assert (len(
benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
def test_cudnn_functions():
"""Test cudnn-function benchmark."""
# Test for default configuration
context = BenchmarkRegistry.create_benchmark_context(
'cudnn-function',
platform=Platform.CUDA,
parameters='--num_warmup 10 --num_steps 10 --num_in_step 100')
assert (BenchmarkRegistry.is_benchmark_context_valid(context))
benchmark = BenchmarkRegistry.launch_benchmark(context)
# Check basic information.
assert (benchmark)
assert (benchmark.name == 'cudnn-function')
assert (benchmark.type == BenchmarkType.MICRO)
# Check parameters specified in BenchmarkContext.
assert (benchmark._args.num_warmup == 10)
assert (benchmark._args.num_steps == 10)
assert (benchmark._args.num_in_step == 100)
# Check results and metrics.
assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS)
assert ('raw_output_0' in benchmark.raw_data)
assert (len(benchmark.raw_data['raw_output_0']) == 1)
assert (isinstance(benchmark.raw_data['raw_output_0'][0], str))
assert (18 <= len(benchmark.result))
for metric in list(benchmark.result.keys()):
assert (len(benchmark.result[metric]) == 1)
assert (isinstance(benchmark.result[metric][0], numbers.Number))
if metric != 'return_code':
assert (len(
benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
# Test for custom configuration
custom_config_str = '{"algo":0,"arrayLength":2,"convType":0,"dilationA":[1,1],"filterStrideA":[1,1],' \
+ '"filterDims":[32,128,3,3],"inputDims":[32,128,14,14],"inputStride":[25088,196,14,1],"inputType":0,'\
+ '"mode":1,"name":"cudnnConvolutionBackwardFilter","outputDims":[32,32,14,14],'\
+ '"outputStride":[6272,196,14,1],"padA":[1,1],"tensorOp":false}'
context = BenchmarkRegistry.create_benchmark_context(
'cudnn-function',
platform=Platform.CUDA,
parameters=
'--num_warmup 10 --num_steps 10 --num_in_step 100 --config_json_str ' +
custom_config_str)
assert (BenchmarkRegistry.is_benchmark_context_valid(context))
benchmark = BenchmarkRegistry.launch_benchmark(context)
# Check basic information.
assert (benchmark)
assert (benchmark.name == 'cudnn-function')
assert (benchmark.type == BenchmarkType.MICRO)
# Check parameters specified in BenchmarkContext.
assert (benchmark._args.num_warmup == 10)
assert (benchmark._args.num_steps == 10)
assert (benchmark._args.num_in_step == 100)
# Check results and metrics.
assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS)
assert ('raw_output_0' in benchmark.raw_data)
assert (len(benchmark.raw_data['raw_output_0']) == 1)
assert (isinstance(benchmark.raw_data['raw_output_0'][0], str))
assert (1 + benchmark.default_metric_count == len(benchmark.result))
for metric in list(benchmark.result.keys()):
assert (len(benchmark.result[metric]) == 1)
assert (isinstance(benchmark.result[metric][0], numbers.Number))
if metric != 'return_code':
assert (len(
benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
