def run_pytorch_tests(self, test_config): """Runs all pytorch based tests. Args: test_config: Config representing the tests to run. """ # Clone the pytorch examples repo. self._git_clone('https://github.com/pytorch/examples.git', os.path.join(self.git_repo_base, 'pytorch_examples')) bench_home = os.path.join(self.git_repo_base, 'pytorch_examples') # pylint: disable=C6204 import torch test_config['framework_version'] = torch.__version__ test_config['framework_describe'] = torch.__version__ # pylint: disable=C6204 from test_runners.pytorch import runner tested = self.check_if_run(test_config, 'pytorch') if not tested: # Calls pytorch runner with lists of tests from the test_config run = runner.TestRunner(os.path.join(self.logs_dir, 'pytorch'), bench_home, auto_test_config=test_config) run.run_tests(test_config['pytorch_tests']) self.update_state(test_config, 'pytorch') else: print('Setup already tested for {} on {}'.format( 'pytorch', test_config))
def run_mxnet_tests(self, test_config): """Runs all MXNet based tests. Args: test_config: Config representing the tests to run. """ # Clone the mxnet repo so we know where it is. self._git_clone('https://github.com/apache/incubator-mxnet.git', os.path.join(self.git_repo_base, 'mxnet_repo')) bench_home = os.path.join(self.git_repo_base, 'mxnet_repo/example/image-classification') # pylint: disable=C6204 import mxnet as mx # pylint: disable=C6204 from test_runners.mxnet import runner test_config['framework_version'] = mx.__version__ test_config['framework_describe'] = mx.__version__ tested = self.check_if_run(test_config, 'mxnet') if not tested: # Calls mxnet runner with lists of tests from the test_config run = runner.TestRunner(os.path.join(self.logs_dir, 'mxnet'), bench_home, auto_test_config=test_config) run.run_tests(test_config['mxnet_tests']) self.update_state(test_config, 'mxnet') else: print('Setup already tested for {} on {}'.format( 'mxnet', test_config))
def check_config_paths(self, config): """Asserts that the methods in the file for tf_models exist.""" f = open(config) auto_config = yaml.safe_load(f) run = runner.TestRunner('/workspace', '/workspace/git/tf_models') if 'tf_models_tests' in auto_config: for test_method in auto_config['tf_models_tests']: ret = getattr(run, test_method, None) self.assertIsNotNone( ret, msg='Method "{}" not found from file "{}"'.format( test_method, config))
def test_resnet50_256_gpu_1_real_fp16(self, _, run_test_suite_mock): """Tests init TestRunner and running a mocked single gpu test.""" run = runner.TestRunner('/workspace', '/workspace/git/tf_models') run.resnet50_256_gpu_1_real_fp16() test_config = run_test_suite_mock.call_args[0][0] # check GPU args and count self.assertEqual(test_config['gpus'], 1) self.assertEqual(test_config['batch_size'], 256) self.assertEqual(test_config['args']['batch_size'], 256) self.assertEqual(test_config['args']['dtype'], 'fp16') self.assertEqual(test_config['test_id'], 'official.resnet50.gpu_1.256.fp16.real') self.assertIn('model', test_config)
def test_resnet50_128_gpu_8_real(self, _, run_test_suite_mock): """Tests init TestRunner and running a mocked single gpu test.""" run = runner.TestRunner('/workspace', '/workspace/git/tf_models') run.resnet50_128_gpu_8_real() test_config = run_test_suite_mock.call_args[0][0] # check GPU args and count self.assertEqual(test_config['gpus'], 8) self.assertEqual(test_config['batch_size'], 128) self.assertEqual(test_config['args']['data_dir'], '/data/imagenet') self.assertEqual(test_config['args']['batch_size'], 8*128) self.assertEqual(test_config['args']['resnet_version'], 1) self.assertEqual(test_config['args']['dtype'], 'fp32') self.assertEqual(test_config['test_id'], 'official.resnet50.gpu_8.128.real') self.assertIn('model', test_config)
def _keras_tf_model_bench(self, auto_config): """Runs keras tf model benchmarks. Args: auto_config: Configuration for running tf_model_bench tests. """ # Module is loaded by this module. # pylint: disable=C6204 import test_runners.keras_tf_models.runner as runner bench_home = os.path.join(self.git_repo_base, 'tf_models') # call tf_garden runner with lists of tests from the test_config run = runner.TestRunner(os.path.join(self.logs_dir, 'keras_tf_models'), bench_home, auto_test_config=auto_config) run.run_tests(auto_config['keras_tf_models_tests'])