def validate(mlcube: DictConfig) -> None:
mlcube.runner = OmegaConf.merge(Config.DEFAULT, mlcube.runner)
Validate(mlcube.runner, 'runner')\
.check_unknown_keys(Config.DEFAULT.keys())\
.check_values(['host', 'platform', 'remote_root'], str, blanks=False)\
.check_values(['interpreter', 'authentication'], DictConfig)
PythonInterpreter.get(mlcube.runner.interpreter).validate(
mlcube.runner.interpreter)
def test_system_interpreter_user_config(self) -> None:
config = {
'type': 'system',
'python': 'python3.8',
'requirements': 'click==7.1.2 mlcube==0.2.2'
}
self.check_state(config, PythonInterpreter.create(config))
def test_system_interpreter_default_config(self) -> None:
self.check_state(
{
'type': 'system',
'python': 'python',
'requirements': ''
}, PythonInterpreter.create({'type': 'system'}))
def run(self) -> None:
conn: t.Text = self.get_connection_string()
remote_env: PythonInterpreter = PythonInterpreter.create(
self.mlcube.runner.interpreter)
# The 'remote_path' variable points to the MLCube root directory on remote host.
remote_path: t.Text = os.path.join(
self.mlcube.runner.remote_root,
os.path.basename(self.mlcube.runtime.root))
try:
cmd = f"mlcube run --mlcube=. --platform={self.mlcube.runner.platform} --task={self.task}"
Shell.ssh(
conn,
f'{remote_env.activate_cmd(noop=":")} && cd {remote_path} && {cmd}'
)
except ExecutionError as err:
raise ExecutionError.mlcube_run_error(
self.__class__.__name__,
f"Error occurred while running MLCube task (name={self.task}).",
**err.context)
# Sync back results
try:
# TODO: Only workspace/ directory is synced. Better solution?
Shell.rsync_dirs(source=f'{conn}:{remote_path}/workspace/',
dest=f'{self.mlcube.runtime.root}/workspace/')
except ExecutionError as err:
raise ExecutionError.mlcube_run_error(
self.__class__.__name__,
"Error occurred while syncing workspace.", **err.context)
def test_virtualenv_interpreter_user_config(self):
config = {
'type': 'virtualenv',
'python': 'python3.8',
'requirements': 'click==7.1.2 mlcube==0.2.2',
'location': '/opt/mlcube_resources/environments',
'name': 'docker_runner-0.2.2'
}
self.check_state(config, PythonInterpreter.create(config))
def test_virtualenv_interpreter_default_config(self):
self.check_state(
{
'type': 'virtualenv',
'python': 'python',
'requirements': '',
'location': StandardPaths.ENVIRONMENTS,
'name': 'MY_NAME'
},
PythonInterpreter.create({
'type': 'virtualenv',
'name': 'MY_NAME'
}))
def configure(self) -> None:
"""Run 'configure' phase for SHH runner."""
conn: t.Text = self.get_connection_string()
remote_env: PythonInterpreter = PythonInterpreter.create(
self.mlcube.runner.interpreter)
# If required, create and configure python environment on remote host
try:
Shell.ssh(conn, remote_env.create_cmd())
except ExecutionError as err:
raise ExecutionError.mlcube_configure_error(
self.__class__.__name__,
f"Error occurred while creating remote python environment (env={remote_env}).",
**err.context)
try:
Shell.ssh(conn, remote_env.configure_cmd())
except ExecutionError as err:
raise ExecutionError.mlcube_configure_error(
self.__class__.__name__,
f"Error occurred while configuring remote python environment (env={remote_env}).",
**err.context)
# The 'local_path' and 'remote_path' must both be directories.
try:
local_path: str = self.mlcube.runtime.root
remote_path: str = os.path.join(self.mlcube.runner.remote_root,
os.path.basename(local_path))
Shell.ssh(conn, f'mkdir -p {remote_path}')
Shell.rsync_dirs(source=f'{local_path}/',
dest=f'{conn}:{remote_path}/')
except ExecutionError as err:
raise ExecutionError.mlcube_configure_error(
self.__class__.__name__,
"Error occurred while syncing local and remote folders.",
**err.context)
# Configure remote MLCube runner. Idea is that we use chain of runners, for instance, SHH Runner -> Docker
# runner. So, the runner to be used on a remote host must configure itself.
try:
cmd = f"mlcube configure --mlcube=. --platform={self.mlcube.runner.platform}"
Shell.ssh(
conn,
f'{remote_env.activate_cmd(noop=":")} && cd {remote_path} && {cmd}'
)
except ExecutionError as err:
raise ExecutionError.mlcube_configure_error(
self.__class__.__name__,
"Error occurred while configuring MLCube on a remote machine.",
**err.context)