def test_run_test_2(self, _, run_test_suite_mock):
"""Tests running list of tests by name and counts how many are run."""
run = runner.TestRunner('/workspace', '/workspace/git/mxnet_git')
tests = ['renset50v1_128_gpu_8_fp16', 'renset50v1_128_gpu_8']
run.run_tests(tests)
tests_run = run_test_suite_mock.call_count
self.assertEqual(tests_run, len(tests))
def main():
parser = argparse.ArgumentParser(description='...')
parser.add_argument('--url', default='http://www.google.com')
parser.add_argument('--debug',
default=runner.TestRunner.options['debug'],
action='store_true')
parser.add_argument('--num_threads',
default=runner.TestRunner.options['num_threads'],
type=int)
parser.add_argument('--out_file',
default=runner.TestRunner.options['out_file'])
parser.add_argument('--num_runs',
default=runner.TestRunner.options['num_runs'],
type=int)
parser.add_argument('--start_delay',
default=runner.TestRunner.options['start_delay'],
type=int)
parser.add_argument('--run_delay',
default=runner.TestRunner.options['run_delay'],
type=int)
args = vars(parser.parse_args())
args['test_class'] = SimpleTestTransaction
runner.TestRunner(args)
def test_8gpu_fp16_batch128(self, _, run_test_suite_mock):
"""Tests running a multi-gpu test and overriding the dtype."""
run = runner.TestRunner('/workspace', '/workspace/git/mxnet_git')
run.renset50v1_128_gpu_8_fp16()
test_config = run_test_suite_mock.call_args[0][0]
# check GPU args and count
self.assertEqual(test_config['gpus'], 8)
self.assertEqual(test_config['batch_size'], 128)
self.assertEqual(test_config['args']['dtype'], 'float16')
self.assertEqual(test_config['args']['gpus'], '0,1,2,3,4,5,6,7')
self.assertEqual(test_config['args']['batch-size'], 128 * 8)
self.assertEqual(test_config['test_id'], 'resnet50v1.gpu_8.128.fp16')
def test_resnet_1gpu_fp32(self, _, run_test_suite_mock):
"""Tests init TestRunner and running a mocked single gpu test."""
run = runner.TestRunner('/workspace', '/workspace/git/tf_models')
run.renset50_32_gpu_1_real()
test_config = run_test_suite_mock.call_args[0][0]
# check GPU args and count
self.assertEqual(test_config['gpus'], 1)
self.assertEqual(test_config['batch_size'], 32)
self.assertEqual(test_config['args']['batch-size'], 32)
self.assertEqual(test_config['test_id'], 'resnet50.gpu_1.32.real')
self.assertIn('model', test_config)
print(test_config['pycmd'])
def test_base_imagenet_args(self, _):
"""Tests base imagenet_args are generated as expected."""
test_config = {}
test_config['data_threads'] = 16
run = runner.TestRunner('/workspace',
'/workspace/git/mxnet_git',
auto_test_config=test_config,
imagenet_dir='/data/mxnet/imagenet/data')
args = run._base_imagenet_args()
self.assertEqual(args['data-train'], '/data/mxnet/imagenet/data')
self.assertEqual(args['data-nthreads'], 16)
def test_1gpu_fp32(self, _, run_test_suite_mock):
"""Tests init TestRunner and running a mocked single gpu test."""
run = runner.TestRunner('/workspace', '/workspace/git/mxnet_git')
run.renset50v1_32_gpu_1()
test_config = run_test_suite_mock.call_args[0][0]
# check GPU args and count
self.assertEqual(test_config['gpus'], 1)
self.assertEqual(test_config['batch_size'], 32)
self.assertEqual(test_config['args']['batch-size'], 32)
self.assertEqual(test_config['args']['dtype'], 'float32')
self.assertEqual(test_config['args']['gpus'], '0')
self.assertEqual(test_config['test_id'], 'resnet50v1.gpu_1.32')
self.assertIn('model', test_config)
def test_resnet50_128_gpu_8_fp16(self, _, run_test_suite_mock):
"""Tests init TestRunner and running a mocked single gpu test."""
run = runner.TestRunner('/workspace', '/workspace/git/tf_models')
run.resnet50_128_gpu_8_fp16()
test_config = run_test_suite_mock.call_args[0][0]
# check GPU args and count
self.assertEqual(test_config['gpus'], 8)
self.assertEqual(test_config['batch_size'], 128)
self.assertEqual(test_config['args']['batch_size'], 1024)
self.assertEqual(test_config['args']['dtype'], 'fp16')
self.assertEqual(test_config['args']['use_synthetic_data'], '')
self.assertEqual(test_config['test_id'],
'garden.resnet50.gpu_8.128.fp16')
self.assertIn('model', test_config)
def test_resnet50v2_64_gpu_1_real(self, _, run_test_suite_mock):
"""Tests init TestRunner and running a mocked single gpu test."""
run = runner.TestRunner('/workspace', '/workspace/git/tf_models')
run.resnet50v2_64_gpu_1_real()
test_config = run_test_suite_mock.call_args[0][0]
# check GPU args and count
self.assertEqual(test_config['gpus'], 1)
self.assertEqual(test_config['batch_size'], 64)
self.assertEqual(test_config['args']['data_dir'], '/data/imagenet')
self.assertEqual(test_config['args']['batch_size'], 64)
self.assertEqual(test_config['args']['resnet_version'], 2)
self.assertEqual(test_config['args']['dtype'], 'fp32')
self.assertEqual(test_config['test_id'],
'garden.resnet50v2.gpu_1.64.real')
self.assertIn('model', test_config)
def test_resnet_8gpu_fp32_cmd_builder(self, _, run_test_suite_mock):
"""Tests init TestRunner and running a mocked single gpu test."""
run = runner.TestRunner('/workspace', '/workspace/git/tf_models')
run.renset50_64_gpu_8_real()
test_config = run_test_suite_mock.call_args[0][0]
# check GPU args and count
self.assertEqual(test_config['gpus'], 8)
self.assertEqual(test_config['batch_size'], 64)
self.assertEqual(test_config['args']['batch-size'], 8 * 64)
self.assertEqual(test_config['test_id'], 'resnet50.gpu_8.64.real')
self.assertIn('model', test_config)
cmd = run._cmd_builder(test_config)
# Verifies the full command.
self.assertEqual(
cmd, 'CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7 python main.py '
'--arch resnet50 --batch-size 512 --epochs 1 '
'--print-freq 1 --workers 5 /data/pytorch/imagenet')
def test_base_imagenet_args_no_data(self, _):
"""Tests base imagenet_args throws an assertion error."""
run = runner.TestRunner('/workspace',
'/workspace/git/mxnet_git',
imagenet_dir=None)
self.assertRaises(AssertionError, run._base_imagenet_args)