def test_run_module(log, popen, entry_point_type_module): with pytest.raises(errors.ExecuteUserScriptError): process.ProcessRunner("module.py", ["--lr", "13"], {}).run() cmd = [sys.executable, "-m", "module", "--lr", "13"] popen.assert_called_with(cmd, cwd=environment.code_dir, env=os.environ, stderr=None) log.assert_called_with(cmd, {})
def test_run_bash(log, popen, entry_point_type_script): with pytest.raises(errors.ExecuteUserScriptError): process.ProcessRunner("launcher.sh", ["--lr", "1 3"], {}).run() cmd = ["/bin/sh", "-c", "./launcher.sh --lr '1 3'"] popen.assert_called_with(cmd, cwd=environment.code_dir, env=os.environ, stderr=None) log.assert_called_with(cmd, {})
def _get_by_runner_type(identifier, user_entry_point=None, args=None, env_vars=None, extra_opts=None): env = environment.Environment() user_entry_point = user_entry_point or env.user_entry_point args = args or env.to_cmd_args() env_vars = env_vars or env.to_env_vars() mpi_args = extra_opts or {} # Default to single process for CPU default_processes_per_host = int( env.num_gpus) if int(env.num_gpus) > 0 else 1 processes_per_host = _mpi_param_value(mpi_args, env, params.MPI_PROCESSES_PER_HOST, default_processes_per_host) if identifier is RunnerType.SMDataParallel and env.is_master: custom_mpi_options = _mpi_param_value( mpi_args, env, params.SMDATAPARALLEL_CUSTOM_MPI_OPTIONS, "") return smdataparallel.SMDataParallelRunner( user_entry_point, args, env_vars, processes_per_host, env.master_hostname, env.distribution_hosts, custom_mpi_options, env.network_interface_name, ) elif identifier is RunnerType.SMDataParallel: return mpi.WorkerRunner(user_entry_point, args, env_vars, processes_per_host, env.master_hostname) elif identifier is RunnerType.MPI and env.is_master: num_processes = _mpi_param_value(mpi_args, env, params.MPI_NUM_PROCESSES) custom_mpi_options = _mpi_param_value(mpi_args, env, params.MPI_CUSTOM_OPTIONS, "") return mpi.MasterRunner( user_entry_point, args, env_vars, processes_per_host, env.master_hostname, env.distribution_hosts, custom_mpi_options, env.network_interface_name, num_processes=num_processes, ) elif identifier is RunnerType.MPI: return mpi.WorkerRunner(user_entry_point, args, env_vars, processes_per_host, env.master_hostname) elif identifier is RunnerType.Process: return process.ProcessRunner(user_entry_point, args, env_vars, processes_per_host) else: raise ValueError("Invalid identifier %s" % identifier)
def test_run_python(log, popen, entry_point_type_script): popen().communicate.return_value = (None, 0) with pytest.raises(errors.ExecuteUserScriptError): process.ProcessRunner("launcher.py", ["--lr", "13"], {}).run(capture_error=True) cmd = [sys.executable, "launcher.py", "--lr", "13"] popen.assert_called_with(cmd, cwd=environment.code_dir, env=os.environ, stderr=subprocess.PIPE) log.assert_called_with(cmd, {})
def test_run_python(log, async_shell, async_gather, entry_point_type_script, event_loop): async_gather.return_value = ("stdout", "stderr") with pytest.raises(errors.ExecuteUserScriptError): rc, output, proc = process.ProcessRunner("launcher.py", ["--lr", "13"], {}, 2).run(capture_error=True) assert output == "stderr" cmd = [sys.executable, "launcher.py", "--lr", "13"] async_shell.assert_called_once() async_gather.assert_called_once() async_shell.assert_called_with( " ".join(cmd), cwd=environment.code_dir, env=os.environ, stderr=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, ) log.assert_called_with(cmd, {})
def test_run_error(): with pytest.raises(errors.ExecuteUserScriptError) as e: process.ProcessRunner("wrong module", [], {}, 1).run() message = str(e.value) assert "ExecuteUserScriptError:" in message