def test_run_tests_pytorch(self, check_state, save_state, pytorch_runner):
"""Tests run_tests for pytorch with run tracker True."""
mock_git_clone, mock_make_dirs, mock_gpu_info, _ = self._patch_logs_git_gpu(
)
self._patch_pytorch(version='0.9')
benchmark_runner = controller.BenchmarkRunner(
'/workspace', 'test_configs/pytorch_config.yaml', framework='pytorch')
check_state.return_value = False
benchmark_runner.run_tests()
check_state.assert_called()
mock_git_clone.assert_called()
mock_make_dirs.assert_called()
mock_gpu_info.assert_called()
call_args = pytorch_runner.call_args
arg0 = call_args[0]
self.assertEqual(arg0[0], '/workspace/logs/pytorch')
self.assertEqual(arg0[1], '/workspace/git/pytorch_examples')
self.assertEqual(len(call_args[1]['auto_test_config']['pytorch_tests']), 4)
saved_object = save_state.call_args[0][1]
key = tracker._hash_key('pytorch', 'FINAL', 'OTB-GPU', '0.9')
entry = saved_object[key]
self.assertIn('pytorch', entry['tests'])
def test_run_tests_mxnet(self, check_state, save_state, mxnet_runner):
"""Tests run_tests for mxnet with run tracker True."""
mock_git_clone, mock_make_dirs, mock_gpu_info, _ = self._patch_logs_git_gpu(
)
self._patch_mxnet()
benchmark_runner = controller.BenchmarkRunner(
'/workspace', 'test_configs/mxnet_config.yaml', framework='mxnet')
check_state.return_value = False
benchmark_runner.run_tests()
check_state.assert_called()
mock_git_clone.assert_called()
mock_make_dirs.assert_called()
mock_gpu_info.assert_called()
call_args = mxnet_runner.call_args
arg0 = call_args[0]
self.assertEqual(arg0[0], '/workspace/logs/mxnet')
self.assertEqual(arg0[1],
'/workspace/git/mxnet_repo/example/image-classification')
self.assertEqual(len(call_args[1]['auto_test_config']['mxnet_tests']), 4)
saved_object = save_state.call_args[0][1]
key = tracker._hash_key('mxnet', 'FINAL', 'OTB-GPU', '1_0_0')
entry = saved_object[key]
self.assertIn('mxnet', entry['tests'])
def _mock_state_object(self, framework, channel, build_type, tests):
"""Returns state object for testing."""
state_object = {}
entry_1 = {}
entry_1['framework'] = framework
entry_1['channel'] = channel
entry_1['build_type'] = build_type
entry_1['tests'] = tests
entry_key_1 = tracker._hash_key('tensorflow', 'RC', 'OTB.GPU')
state_object[entry_key_1] = entry_1
return state_object
def _mock_state_object(self):
"""Returns state object for testing."""
state_object = {}
entry_1 = {}
entry_1['framework'] = 'tensorflow'
entry_1['channel'] = 'RC'
entry_1['build_type'] = 'OTB.GPU'
entry_1['version'] = '1.4_02102018'
entry_1['tests'] = ['foobar']
entry_key_1 = tracker._hash_key('tensorflow', 'RC', 'OTB.GPU',
'1.4_02102018')
state_object[entry_key_1] = entry_1
entry_2 = {}
entry_2['channel'] = 'FINAL'
entry_2['build_type'] = 'OTB.GPU'
entry_1['version'] = '1_0_0'
entry_2['tests'] = ['mxnet']
entry_key_2 = tracker._hash_key('mxnet', 'FINAL', 'OTB.GPU', '1_0_0')
state_object[entry_key_2] = entry_2
return state_object
def test_update_state_new_key(self, mock_get_state_object,
mock_save_state_object):
"""Tests update state when new key needs to be created."""
mock_get_state_object.return_value = self._mock_state_object()
tracker.update_state(None, 'tensorflow', 'FINAL', 'OTB.GPU',
'1.4_02102018', 'tf_models')
saved_object = mock_save_state_object.call_args[0][1]
hash_key = tracker._hash_key('tensorflow', 'FINAL', 'OTB.GPU',
'1.4_02102018')
self.assertIn(hash_key, saved_object)
self.assertIn('tf_models', saved_object[hash_key]['tests'])
def test_run_tests(self, clone_repos_mock, store_repo_info, tf_cnn_bench,
get_tf_full_version, check_state, save_state):
"""Tests run_tests where TensorFlow based tests are run."""
_, mock_make_dirs, mock_gpu_info, mock_cpu_info = self._patch_logs_git_gpu()
benchmark_runner = controller.BenchmarkRunner(
'/workspace',
'test_configs/basic_test_config.yaml',
framework='tensorflow')
expected_version = ['1.5RC0-dev20171001', 'v1.3.0-rc1-2884-g2d5b76169']
get_tf_full_version.return_value = expected_version
check_state.return_value = False
benchmark_runner.run_tests()
check_state.assert_called()
clone_repos_mock.assert_called()
mock_make_dirs.assert_called()
store_repo_info.assert_called()
mock_gpu_info.assert_called()
mock_cpu_info.assert_called()
self.assertEqual(os.environ['GOOGLE_APPLICATION_CREDENTIALS'],
'/auth_tokens/tensorflow_performance_upload_tb.json')
self.assertIn('/workspace/git/benchmark_harness/oss_bench', sys.path)
# needed for tf_models tests.
self.assertEqual(os.environ['PYTHONPATH'], '/workspace/git/tf_models')
arg0 = tf_cnn_bench.call_args[0][0]
self.assertEqual(arg0['accel_type'], 'GTX 970')
self.assertEqual(arg0['gpu_driver'], 387.11)
self.assertEqual(arg0['framework_version'], expected_version[0])
self.assertEqual(arg0['framework_describe'], expected_version[1])
self.assertIn('model_name', arg0['cpu_info'])
saved_object = save_state.call_args[0][1]
key = tracker._hash_key('tensorflow', 'NIGHTLY', 'OTB-GPU',
expected_version[1])
entry = saved_object[key]
self.assertIn('tf_cnn_bench', entry['tests'])