def _check_mlflow():
"""
Conda
:return:
"""
check_error([
'conda', 'run', '-n', const.CONDA_TRAINING_ENV, 'mlflow', '--version'
],
MLFlowIsNotInstalledInConda,
capture_error=True)
def _update_codna_env(ml_project_dir: str):
ml_project_file = _find_mlproject_file_path(ml_project_dir)
conda_fp = _extract_conda_file_name(ml_project_file)
abs_conda_fp = join(ml_project_dir, conda_fp)
logger.info(
f'Found MLproject conda file with dependencies: {abs_conda_fp}')
logger.info(
f'Start to update {const.CONDA_TRAINING_ENV} conda env using {conda_fp} file'
)
check_error([
'conda', 'env', 'update', '-n', const.CONDA_TRAINING_ENV, '-f',
abs_conda_fp
],
_CalledProcessError,
capture_error=True)
def install(path, capture_error=False): # type: (str, bool) -> None
"""Install a Python module in the executing Python environment.
Args:
path (str): Real path location of the Python module.
capture_error (bool): Default false. If True, the running process captures the
stderr, and appends it to the returned Exception message in case of errors.
"""
cmd = "%s -m pip install . " % _process.python_executable()
if has_requirements(path):
cmd += "-r requirements.txt"
logger.info("Installing module with the following command:\n%s", cmd)
_process.check_error(
shlex.split(cmd), _errors.InstallModuleError, cwd=path, capture_error=capture_error
)
def _call(user_entry_point, args=None, env_vars=None, wait=True, capture_error=False):
# type: (str, list, dict, bool, bool) -> Popen
args = args or []
env_vars = env_vars or {}
entrypoint_type = entry_point_type(_env.code_dir, user_entry_point)
if entrypoint_type is EntryPointType.PYTHON_PACKAGE:
cmd = [_process.python_executable(), '-m', user_entry_point.replace('.py', '')] + args
elif entrypoint_type is EntryPointType.PYTHON_PROGRAM:
cmd = [_process.python_executable(), user_entry_point] + args
else:
cmd = ['/bin/sh', '-c', './%s %s' % (user_entry_point, ' '.join(args))]
_logging.log_script_invocation(cmd, env_vars)
if wait:
return _process.check_error(cmd, _errors.ExecuteUserScriptError, capture_error=capture_error)
else:
return _process.create(cmd, _errors.ExecuteUserScriptError, capture_error=capture_error)
def run(module_name, args=None, env_vars=None, wait=True, capture_error=False):
# type: (str, list, dict, bool, bool) -> subprocess.Popen
"""Run Python module as a script.
Search sys.path for the named module and execute its contents as the __main__ module.
Since the argument is a module name, you must not give a file extension (.py). The module name
should be a valid absolute Python module name, but the implementation may not always enforce
this (e.g. it may allow you to use a name that includes a hyphen).
Package names (including namespace packages) are also permitted. When a package name is supplied
instead of a normal module, the interpreter will execute <pkg>.__main__ as the main module. This
behaviour is deliberately similar to the handling of directories and zipfiles that are passed to
the interpreter as the script argument.
Note This option cannot be used with built-in modules and extension modules written in C, since
they do not have Python module files. However, it can still be used for precompiled modules,
even if the original source file is not available. If this option is given, the first element
of sys.argv will be the full path to the module file (while the module file is being located,
the first element will be set to "-m"). As with the -c option, the current directory will be
added to the start of sys.path.
You can find more information at https://docs.python.org/3/using/cmdline.html#cmdoption-m
Example:
>>>import sagemaker_containers
>>>from sagemaker_containers.beta.framework import mapping, modules
>>>env = sagemaker_containers.training_env()
{'channel-input-dirs': {'training': '/opt/ml/input/training'},
'model_dir': '/opt/ml/model', ...}
>>>hyperparameters = env.hyperparameters
{'batch-size': 128, 'model_dir': '/opt/ml/model'}
>>>args = mapping.to_cmd_args(hyperparameters)
['--batch-size', '128', '--model_dir', '/opt/ml/model']
>>>env_vars = mapping.to_env_vars()
['SAGEMAKER_CHANNELS':'training', 'SAGEMAKER_CHANNEL_TRAINING':'/opt/ml/input/training',
'MODEL_DIR':'/opt/ml/model', ...}
>>>modules.run('user_script', args, env_vars)
SAGEMAKER_CHANNELS=training SAGEMAKER_CHANNEL_TRAINING=/opt/ml/input/training \
SAGEMAKER_MODEL_DIR=/opt/ml/model python -m user_script --batch-size 128
--model_dir /opt/ml/model
Args:
module_name (str): module name in the same format required by python -m <module-name>
cli command.
args (list): A list of program arguments.
env_vars (dict): A map containing the environment variables to be written.
capture_error (bool): Default false. If True, the running process captures the
stderr, and appends it to the returned Exception message in case of errors.
"""
args = args or []
env_vars = env_vars or {}
cmd = [_process.python_executable(), "-m", module_name] + args
_logging.log_script_invocation(cmd, env_vars)
if wait:
return _process.check_error(cmd,
_errors.ExecuteUserScriptError,
capture_error=capture_error)
else:
return _process.create(cmd,
_errors.ExecuteUserScriptError,
capture_error=capture_error)
def test_check_error(popen):
process = MagicMock(wait=MagicMock(return_value=0))
popen.return_value = process
assert process == _process.check_error(["run"], _errors.ExecuteUserScriptError)
def test_gethostname_with_env_not_set(opt_ml_input_config):
py_cmd = "import gethostname\nassert gethostname.call(30) == 'algo-9'"
with pytest.raises(_errors.ExecuteUserScriptError):
_process.check_error([sys.executable, "-c", py_cmd],
_errors.ExecuteUserScriptError)
def _check_conda():
check_error(['conda', '--version'],
CondaIsNotInstalled,
capture_error=True)
def _check_training_env():
check_error(
['conda', 'run', '-n', const.CONDA_TRAINING_ENV, 'conda', 'info'],
CondaTrainingEnvIsNotCreated,
capture_error=True)
