def test_run_module(log, popen, entry_point_type_module): with pytest.raises(_errors.ExecuteUserScriptError): _process.ProcessRunner("module.py", ["--lr", "13"], {}).run() cmd = [sys.executable, "-m", "module", "--lr", "13"] popen.assert_called_with(cmd, cwd=_env.code_dir, env=os.environ, stderr=None) log.assert_called_with(cmd, {})
def test_run_bash(log, popen, entry_point_type_script): with pytest.raises(_errors.ExecuteUserScriptError): _process.ProcessRunner("launcher.sh", ["--lr", "1 3"], {}).run() cmd = ["/bin/sh", "-c", "./launcher.sh --lr '1 3'"] popen.assert_called_with(cmd, cwd=_env.code_dir, env=os.environ, stderr=None) log.assert_called_with(cmd, {})
def test_run_python(log, popen, entry_point_type_script): popen().communicate.return_value = (None, 0) with pytest.raises(_errors.ExecuteUserScriptError): _process.ProcessRunner("launcher.py", ["--lr", "13"], {}).run(capture_error=True) cmd = [sys.executable, "launcher.py", "--lr", "13"] popen.assert_called_with(cmd, cwd=_env.code_dir, env=os.environ, stderr=subprocess.PIPE) log.assert_called_with(cmd, {})
def test_run_bash(log, popen, entry_point_type_script): with pytest.raises(_errors.ExecuteUserScriptError): _process.ProcessRunner('launcher.sh', ['--lr', '13'], {}).run() cmd = ['/bin/sh', '-c', './launcher.sh --lr 13'] popen.assert_called_with(cmd, cwd=_env.code_dir, env=os.environ, stderr=None) log.assert_called_with(cmd, {})
def _get_by_runner_type(identifier, user_entry_point=None, args=None, env_vars=None, extra_opts=None): """Placeholder docstring""" env = sagemaker_containers.training_env() user_entry_point = user_entry_point or env.user_entry_point args = args or env.to_cmd_args() env_vars = env_vars or env.to_env_vars() if identifier is RunnerType.MPI and env.is_master: mpi_args = extra_opts or {} # Default to single process for CPU default_processes_per_host = env.num_gpus if env.num_gpus > 0 else 1 processes_per_host = _mpi_param_value(mpi_args, env, _params.MPI_PROCESSES_PER_HOST, default_processes_per_host) num_processes = _mpi_param_value(mpi_args, env, _params.MPI_NUM_PROCESSES) custom_mpi_options = _mpi_param_value(mpi_args, env, _params.MPI_CUSTOM_OPTIONS, "") return _mpi.MasterRunner( user_entry_point, args, env_vars, env.master_hostname, env.hosts, processes_per_host, custom_mpi_options, env.network_interface_name, num_processes=num_processes, ) elif identifier is RunnerType.MPI: return _mpi.WorkerRunner(user_entry_point, args, env_vars, env.master_hostname) elif identifier is RunnerType.Process: return _process.ProcessRunner(user_entry_point, args, env_vars) else: raise ValueError("Invalid identifier %s" % identifier)
def test_run_python_capture_error(log, popen, entry_point_type_script): mock_process = MagicMock() mock_process.stdout.readline.return_value = b"stdout" mock_process.stderr.readline.return_value = b"stderr" mock_process.stdout.read.return_value = b"stdout" mock_process.stderr.read.return_value = b"stderr" mock_process.poll.return_value = 1 popen.return_value = mock_process with pytest.raises(_errors.ExecuteUserScriptError): _process.ProcessRunner("launcher.py", ["--lr", "13"], {}).run(capture_error=True) cmd = [sys.executable, "launcher.py", "--lr", "13"] popen.assert_called_with(cmd, cwd=_env.code_dir, env=os.environ, stdout=subprocess.PIPE, stderr=subprocess.PIPE) log.assert_called_with(cmd, {})
def _get_by_runner_type(identifier): env = sagemaker_containers.training_env() if identifier is RunnerType.MPI and env.is_master: processes_per_host = env.additional_framework_parameters.get( _params.MPI_PROCESSES_PER_HOST, 1) custom_mpi_options = env.additional_framework_parameters.get( _params.MPI_CUSTOM_OPTIONS, '') return _mpi.MasterRunner(env.user_entry_point, env.to_cmd_args(), env.to_env_vars(), env.master_hostname, env.hosts, processes_per_host, custom_mpi_options, env.network_interface_name) elif identifier is RunnerType.MPI: return _mpi.WorkerRunner(env.user_entry_point, env.to_cmd_args(), env.to_env_vars(), env.master_hostname) elif identifier is RunnerType.Process: return _process.ProcessRunner(env.user_entry_point, env.to_cmd_args(), env.to_env_vars()) else: raise ValueError('Invalid identifier %s' % identifier)
def _get_by_runner_type(identifier, user_entry_point=None, args=None, env_vars=None, extra_opts=None): env = sagemaker_containers.training_env() user_entry_point = user_entry_point or env.user_entry_point args = args or env.to_cmd_args() env_vars = env_vars or env.to_env_vars() if identifier is RunnerType.MPI and env.is_master: mpi_args = extra_opts or {} processes_per_host = _mpi_param_value(mpi_args, env, _params.MPI_PROCESSES_PER_HOST, 1) num_processes = _mpi_param_value(mpi_args, env, _params.MPI_NUM_PROCESSES) custom_mpi_options = _mpi_param_value(mpi_args, env, _params.MPI_CUSTOM_OPTIONS, '') return _mpi.MasterRunner(user_entry_point, args, env_vars, env.master_hostname, env.hosts, processes_per_host, custom_mpi_options, env.network_interface_name, num_processes=num_processes) elif identifier is RunnerType.MPI: return _mpi.WorkerRunner(user_entry_point, args, env_vars, env.master_hostname) elif identifier is RunnerType.Process: return _process.ProcessRunner(user_entry_point, args, env_vars) else: raise ValueError('Invalid identifier %s' % identifier)
def test_run_error(): with pytest.raises(_errors.ExecuteUserScriptError) as e: _process.ProcessRunner("wrong module", [], {}).run() message = str(e.value) assert "ExecuteUserScriptError:" in message