def run(self, wait=True, capture_error=False):
"""Run the process.
Args:
wait (bool): A boolean indicating whether to wait and check for errors.
Defaults to True.
capture_error (bool): A boolean indicating whether to direct stderr to a stream
that can later be read. Defaults to False.
Returns:
process (subprocess.Popen): The spawned process.
"""
self._setup()
cmd = self._create_command()
cmd.extend(super(SMDataParallelRunner, self)._create_command())
logging_config.log_script_invocation(cmd, self._env_vars)
if wait:
process_spawned = process.check_error(
cmd,
errors.ExecuteUserScriptError,
capture_error=capture_error,
cwd=environment.code_dir,
)
else:
process_spawned = process.create(
cmd,
errors.ExecuteUserScriptError,
capture_error=capture_error,
cwd=environment.code_dir,
)
self._tear_down()
return process_spawned
def test_check_error(popen):
test_process = MagicMock(wait=MagicMock(return_value=0))
popen.return_value = test_process
assert test_process == process.check_error(["run"],
errors.ExecuteUserScriptError,
1,
capture_error=False)
def install_requirements(path,
capture_error=False): # type: (str, bool) -> None
"""Install dependencies from requirements.txt in the executing Python environment.
Args:
path (str): Real path location of the requirements.txt file.
capture_error (bool): Default false. If True, the running process captures the
stderr, and appends it to the returned Exception message in case of errors.
"""
cmd = "{} -m pip install -r requirements.txt".format(
process.python_executable())
logger.info(
"Installing dependencies from requirements.txt:\n{}".format(cmd))
process.check_error(shlex.split(cmd),
errors.InstallRequirementsError,
cwd=path,
capture_error=capture_error)
def install(path, capture_error=False): # type: (str, bool) -> None
"""Install a Python module in the executing Python environment.
Args:
path (str): Real path location of the Python module.
capture_error (bool): Default false. If True, the running process captures the
stderr, and appends it to the returned Exception message in case of errors.
"""
cmd = "%s -m pip install . " % process.python_executable()
if has_requirements(path):
cmd += "-r requirements.txt"
logger.info("Installing module with the following command:\n%s", cmd)
process.check_error(shlex.split(cmd),
errors.InstallModuleError,
cwd=path,
capture_error=capture_error)
def run(self, wait=True, capture_error=False):
"""Run the process.
Args:
wait (bool): A boolean indicating whether to wait and check for errors.
Defaults to True.
capture_error (bool): A boolean indicating whether to direct stderr to a stream
that can later be read. Defaults to False.
Returns:
process (subprocess.Popen): The spawned process.
"""
self._setup()
cmd = self._create_command()
logging_config.log_script_invocation(cmd, self._env_vars)
training_env = environment.Environment()
exception_classes = get_modelparallel_exception_classes()
if wait:
process_spawned = process.check_error(
cmd,
exception_classes
if training_env.is_modelparallel_enabled
else errors.ExecuteUserScriptError,
self._processes_per_host,
capture_error=capture_error,
cwd=environment.code_dir,
)
else:
_, _, process_spawned = process.create(
cmd,
exception_classes
if training_env.is_modelparallel_enabled
else errors.ExecuteUserScriptError,
self._processes_per_host,
capture_error=capture_error,
cwd=environment.code_dir,
)
self._tear_down()
return process_spawned
def test_gethostname_with_env_not_set(opt_ml_input_config):
py_cmd = "import gethostname\nassert gethostname.call(30) == 'algo-9'"
with pytest.raises(errors.ExecuteUserScriptError):
process.check_error([sys.executable, "-c", py_cmd],
errors.ExecuteUserScriptError)