def run_pytorch_tests(self, test_config):
"""Runs all pytorch based tests.
Args:
test_config: Config representing the tests to run.
"""
# Clone the pytorch examples repo.
self._git_clone('https://github.com/pytorch/examples.git',
os.path.join(self.git_repo_base, 'pytorch_examples'))
bench_home = os.path.join(self.git_repo_base, 'pytorch_examples')
# pylint: disable=C6204
import torch
test_config['framework_version'] = torch.__version__
test_config['framework_describe'] = torch.__version__
# pylint: disable=C6204
from test_runners.pytorch import runner
tested = self.check_if_run(test_config, 'pytorch')
if not tested:
# Calls pytorch runner with lists of tests from the test_config
run = runner.TestRunner(os.path.join(self.logs_dir, 'pytorch'),
bench_home,
auto_test_config=test_config)
run.run_tests(test_config['pytorch_tests'])
self.update_state(test_config, 'pytorch')
else:
print('Setup already tested for {} on {}'.format(
'pytorch', test_config))
def run_mxnet_tests(self, test_config):
"""Runs all MXNet based tests.
Args:
test_config: Config representing the tests to run.
"""
# Clone the mxnet repo so we know where it is.
self._git_clone('https://github.com/apache/incubator-mxnet.git',
os.path.join(self.git_repo_base, 'mxnet_repo'))
bench_home = os.path.join(self.git_repo_base,
'mxnet_repo/example/image-classification')
# pylint: disable=C6204
import mxnet as mx
# pylint: disable=C6204
from test_runners.mxnet import runner
test_config['framework_version'] = mx.__version__
test_config['framework_describe'] = mx.__version__
tested = self.check_if_run(test_config, 'mxnet')
if not tested:
# Calls mxnet runner with lists of tests from the test_config
run = runner.TestRunner(os.path.join(self.logs_dir, 'mxnet'),
bench_home,
auto_test_config=test_config)
run.run_tests(test_config['mxnet_tests'])
self.update_state(test_config, 'mxnet')
else:
print('Setup already tested for {} on {}'.format(
'mxnet', test_config))
def check_config_paths(self, config):
"""Asserts that the methods in the file for tf_models exist."""
f = open(config)
auto_config = yaml.safe_load(f)
run = runner.TestRunner('/workspace', '/workspace/git/tf_models')
if 'tf_models_tests' in auto_config:
for test_method in auto_config['tf_models_tests']:
ret = getattr(run, test_method, None)
self.assertIsNotNone(
ret,
msg='Method "{}" not found from file "{}"'.format(
test_method, config))
def test_resnet50_256_gpu_1_real_fp16(self, _, run_test_suite_mock):
"""Tests init TestRunner and running a mocked single gpu test."""
run = runner.TestRunner('/workspace', '/workspace/git/tf_models')
run.resnet50_256_gpu_1_real_fp16()
test_config = run_test_suite_mock.call_args[0][0]
# check GPU args and count
self.assertEqual(test_config['gpus'], 1)
self.assertEqual(test_config['batch_size'], 256)
self.assertEqual(test_config['args']['batch_size'], 256)
self.assertEqual(test_config['args']['dtype'], 'fp16')
self.assertEqual(test_config['test_id'],
'official.resnet50.gpu_1.256.fp16.real')
self.assertIn('model', test_config)
def test_resnet50_128_gpu_8_real(self, _, run_test_suite_mock):
"""Tests init TestRunner and running a mocked single gpu test."""
run = runner.TestRunner('/workspace', '/workspace/git/tf_models')
run.resnet50_128_gpu_8_real()
test_config = run_test_suite_mock.call_args[0][0]
# check GPU args and count
self.assertEqual(test_config['gpus'], 8)
self.assertEqual(test_config['batch_size'], 128)
self.assertEqual(test_config['args']['data_dir'], '/data/imagenet')
self.assertEqual(test_config['args']['batch_size'], 8*128)
self.assertEqual(test_config['args']['resnet_version'], 1)
self.assertEqual(test_config['args']['dtype'], 'fp32')
self.assertEqual(test_config['test_id'], 'official.resnet50.gpu_8.128.real')
self.assertIn('model', test_config)
def _keras_tf_model_bench(self, auto_config):
"""Runs keras tf model benchmarks.
Args:
auto_config: Configuration for running tf_model_bench tests.
"""
# Module is loaded by this module.
# pylint: disable=C6204
import test_runners.keras_tf_models.runner as runner
bench_home = os.path.join(self.git_repo_base, 'tf_models')
# call tf_garden runner with lists of tests from the test_config
run = runner.TestRunner(os.path.join(self.logs_dir, 'keras_tf_models'),
bench_home,
auto_test_config=auto_config)
run.run_tests(auto_config['keras_tf_models_tests'])