def test_run_benchmark(mock_run_command, mock_subprocess, mock_platform, mock_os, mock_chdir, mock_path_exists, mock_is_dir, mock_listdir, mock_rmtree, mock_mkdir, test_args, expected_cmd): """ Runs through executing the specified run_tf_benchmarks.py command from the test_args and verifying that the model_init file calls run_command with the expected_cmd string. """ mock_path_exists.return_value = True mock_is_dir.return_value = True parse_model_args_file() mock_listdir.return_value = True clear_kmp_env_vars() platform_config.set_mock_system_type(mock_platform) platform_config.set_mock_os_access(mock_os) platform_config.set_mock_lscpu_subprocess_values(mock_subprocess) test_arg_list = test_args.split(" ") with patch.object(sys, "argv", test_arg_list): model_benchmark = ModelBenchmarkUtil() model_benchmark.main() assert len(mock_run_command.call_args_list) == 1 call_args = mock_run_command.call_args_list[0][0][0] # python3 argparse parses things in different order than python2 # we'll check that the args are all there though for actual_arg, expected_arg in zip(sorted(call_args.split()), sorted(expected_cmd.split())): # use fnmatch in case we have file names with wildcards (like timestamps in output files) assert fnmatch.fnmatch(actual_arg, expected_arg), \ "Expected: {}\nActual: {}".format(expected_cmd, call_args)
def test_run_benchmark(mock_run_command, mock_subprocess, mock_platform, mock_os, mock_chdir, mock_path_exists, mock_is_dir, mock_listdir, mock_rmtree, mock_mkdir, test_args, expected_cmd): """ Runs through executing the specified run_tf_benchmarks.py command from the test_args and verifying that the model_init file calls run_command with the expected_cmd string. """ mock_path_exists.return_value = True mock_is_dir.return_value = True parse_model_args_file() mock_listdir.return_value = True platform_config.set_mock_system_type(mock_platform) platform_config.set_mock_os_access(mock_os) platform_config.set_mock_lscpu_subprocess_values(mock_subprocess) test_arg_list = test_args.split(" ") with patch.object(sys, "argv", test_arg_list): model_benchmark = ModelBenchmarkUtil() model_benchmark.main() assert len(mock_run_command.call_args_list) == 1 call_args = mock_run_command.call_args_list[0][0][0] # python3 argparse parses things in different order than python2 # we'll check that the args are all there though assert sorted(call_args.split()) == sorted(expected_cmd.split())
def test_run_benchmark(mock_run_command, mock_subprocess, mock_platform, mock_os, mock_get_cpuset, mock_glob, mock_remove, mock_chdir, mock_stat, mock_path_exists, mock_is_file, mock_is_dir, mock_listdir, mock_rmtree, mock_mkdir, test_args, expected_cmd, comment, cpuset): """ Runs through executing the specified run_tf_benchmarks.py command from the test_args and verifying that the model_init file calls run_command with the expected_cmd string. """ print("****** Running The {} test ******".format(comment)) os.environ["PYTHON_EXE"] = "python" if "mpi" not in test_args: os.environ["MPI_NUM_PROCESSES"] = "None" os.environ["MPI_HOSTNAMES"] = "None" else: if "--mpi_num_processes=" in test_args: match_mpi_procs = re.search('--mpi_num_processes=([0-9]+)', test_args) if match_mpi_procs and match_mpi_procs.lastindex >= 1: os.environ["MPI_NUM_PROCESSES"] = match_mpi_procs.group(1) if "--mpi_num_processes_per_socket=" in test_args: match_per_socket = re.search( '--mpi_num_processes_per_socket=([0-9]+)', test_args) if match_per_socket and match_per_socket.lastindex >= 1: os.environ[ "MPI_NUM_PROCESSES_PER_SOCKET"] = match_per_socket.group(1) mock_os.path.exists.side_effect = True mock_get_cpuset.return_value = cpuset mock_is_dir.return_value = True mock_is_file.return_value = True mock_stat.return_value = MagicMock(st_nlink=0) parse_model_args_file() mock_listdir.return_value = ["data.record"] mock_glob.return_value = ["/usr/lib/libtcmalloc.so.4.2.6"] clear_kmp_env_vars() platform_config.set_mock_system_type(mock_platform) platform_config.set_mock_os_access(mock_os) platform_config.set_mock_lscpu_subprocess_values(mock_subprocess) test_args = re.sub( " +", " ", test_args) # get rid of extra spaces in the test_args string expected_cmd = re.sub( " +", " ", expected_cmd) # get rid of extra spaces in the expected_cmd string test_arg_list = test_args.split(" ") with patch.object(sys, "argv", test_arg_list): model_benchmark = ModelBenchmarkUtil() model_benchmark.main() assert len(mock_run_command.call_args_list) == 1 call_args = mock_run_command.call_args_list[0][0][0] # python3 argparse parses things in different order than python2 # we'll check that the args are all there though for actual_arg, expected_arg in zip(sorted(call_args.split()), sorted(expected_cmd.split())): # use fnmatch in case we have file names with wildcards (like timestamps in output files) assert fnmatch.fnmatch(actual_arg, expected_arg), \ "Expected: {}\nActual: {}".format(expected_cmd, call_args)
def test_run_benchmark(mock_run_command, mock_subprocess, mock_platform, mock_os, mock_glob, mock_remove, mock_chdir, mock_stat, mock_path_exists, mock_is_file, mock_is_dir, mock_listdir, mock_rmtree, mock_mkdir, test_args, expected_cmd, comment): """ Runs through executing the specified run_tf_benchmarks.py command from the test_args and verifying that the model_init file calls run_command with the expected_cmd string. """ if comment in [ "tf_ssd_resnet34_args.json :: ssd_resnet34_fp32_training", "tf_gnmt_args.json :: gnmt_fp32_throughput", "tf_gnmt_args.json :: gnmt_fp32_latency" ]: pytest.skip() print("****** Running The {} test ******".format(comment)) os.environ["PYTHON_EXE"] = "python" if "mpi" not in mock_run_command: os.environ["MPI_NUM_PROCESSES"] = "None" os.environ["MPI_HOSTNAMES"] = "None" mock_path_exists.return_value = True mock_is_dir.return_value = True mock_is_file.return_value = True mock_stat.return_value = MagicMock(st_nlink=0) parse_model_args_file() mock_listdir.return_value = ["data.record"] mock_glob.return_value = ["/usr/lib/libtcmalloc.so.4.2.6"] clear_kmp_env_vars() platform_config.set_mock_system_type(mock_platform) platform_config.set_mock_os_access(mock_os) platform_config.set_mock_lscpu_subprocess_values(mock_subprocess) test_args = re.sub( " +", " ", test_args) # get rid of extra spaces in the test_args string expected_cmd = re.sub( " +", " ", expected_cmd) # get rid of extra spaces in the expected_cmd string test_arg_list = test_args.split(" ") with patch.object(sys, "argv", test_arg_list): model_benchmark = ModelBenchmarkUtil() model_benchmark.main() assert len(mock_run_command.call_args_list) == 1 call_args = mock_run_command.call_args_list[0][0][0] # python3 argparse parses things in different order than python2 # we'll check that the args are all there though for actual_arg, expected_arg in zip(sorted(call_args.split()), sorted(expected_cmd.split())): # use fnmatch in case we have file names with wildcards (like timestamps in output files) assert fnmatch.fnmatch(actual_arg, expected_arg), \ "Expected: {}\nActual: {}".format(expected_cmd, call_args)
def test_run_benchmark_bad_socket(mock_run_command, mock_subprocess, mock_platform, mock_os, mock_get_cpuset, mock_glob, mock_remove, mock_chdir, mock_stat, mock_path_exists, mock_is_file, mock_is_dir, mock_listdir, mock_rmtree, mock_mkdir, test_args, socket_id, cpuset): """ Checks to ensure that the proper error handling is done when the cpuset does not include any cores for the specified socket_id """ os.environ["PYTHON_EXE"] = "python" if "mpi" not in test_args: os.environ["MPI_NUM_PROCESSES"] = "None" os.environ["MPI_HOSTNAMES"] = "None" else: if "--mpi_num_processes=" in test_args: match_mpi_procs = re.search('--mpi_num_processes=([0-9]+)', test_args) if match_mpi_procs and match_mpi_procs.lastindex >= 1: os.environ["MPI_NUM_PROCESSES"] = match_mpi_procs.group(1) if "--mpi_num_processes_per_socket=" in test_args: match_per_socket = re.search( '--mpi_num_processes_per_socket=([0-9]+)', test_args) if match_per_socket and match_per_socket.lastindex >= 1: os.environ[ "MPI_NUM_PROCESSES_PER_SOCKET"] = match_per_socket.group(1) mock_os.path.exists.side_effect = True mock_get_cpuset.return_value = cpuset mock_is_dir.return_value = True mock_is_file.return_value = True mock_stat.return_value = MagicMock(st_nlink=0) parse_model_args_file() mock_listdir.return_value = ["data.record"] mock_glob.return_value = ["/usr/lib/libtcmalloc.so.4.2.6"] clear_kmp_env_vars() platform_config.set_mock_system_type(mock_platform) platform_config.set_mock_os_access(mock_os) platform_config.set_mock_lscpu_subprocess_values(mock_subprocess) test_args = re.sub( " +", " ", test_args) # get rid of extra spaces in the test_args string test_arg_list = test_args.split(" ") with pytest.raises( SystemExit, match="ERROR: There are no socket id {} cores in the cpuset.". format(socket_id)): with patch.object(sys, "argv", test_arg_list): model_benchmark = ModelBenchmarkUtil() model_benchmark.main()
def setup_mock_values(mock_platform, mock_os, mock_subprocess): platform_config.set_mock_system_type(mock_platform) platform_config.set_mock_os_access(mock_os) platform_config.set_mock_lscpu_subprocess_values(mock_subprocess)
def setup_mock_values(platform_mock, os_mock, subprocess_mock): platform_config.set_mock_system_type(platform_mock) platform_config.set_mock_os_access(os_mock) platform_config.set_mock_lscpu_subprocess_values(subprocess_mock)