def readme(args):
"""Display the model's README information."""
model = args.model
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
# Setup.
logger = logging.getLogger(__name__)
logger.info("Get README of {}.".format(model))
path = utils.get_package_dir(model)
readme_file = os.path.join(path, README)
# Check that the model is installed.
utils.check_model_installed(model)
# Display the README.
if not os.path.exists(
readme_file): # Try to generate README from README.md
readme_raw = readme_file[:readme_file.rfind('.')] + '.md'
if not os.path.exists(readme_raw):
readme_raw = readme_raw[:readme_raw.rfind('.')] + '.rst'
if not os.path.exists(readme_raw):
raise utils.ModelReadmeNotFoundException(model, readme_file)
script = os.path.join(os.path.dirname(__file__), 'scripts',
'convert_readme.sh')
command = "/bin/bash {} {} {}".format(script, readme_raw, README)
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
if proc.returncode != 0:
errors = errors.decode("utf-8")
command_not_found = re.compile(r"\d: (.*):.*not found").search(
errors)
if command_not_found is not None:
raise utils.LackPrerequisiteException(
command_not_found.group(1))
print("An error was encountered:\n")
print(errors)
raise utils.ModelReadmeNotFoundException(model, readme_file)
with open(readme_file, 'r') as f:
print(utils.drop_newline(f.read()))
# Suggest next step.
if not args.quiet:
utils.print_next_step('readme', model=model)
def remove_model(args):
"""Remove installed model."""
# Setup.
model = args.model
if model is None:
if os.path.exists(MLINIT):
path = MLINIT
msg = "*Completely* remove all installed models in '{}' [y/N]? "
else:
msg = "The local model folder '{}' does not exist. Nothing to do."
msg = msg.format(MLINIT)
print(msg)
sys.exit(1)
else:
path = MLINIT + model
msg = "Remove '{}' [y/N]? "
# Check that the model is installed.
utils.check_model_installed(model)
sys.stdout.write(msg.format(path))
choice = input().lower()
if choice == 'y':
rmtree(path)
else:
if model is None and not args.quiet:
utils.print_next_step('remove')
def remove_model(args):
"""Remove installed model."""
# TODO: Remove .archive and .config for the model.
model = args.model
# Determine if remove all model pkgs or a certain model pkg.
cache = None
if model is None:
path = utils.get_init_dir()
if os.path.exists(path):
msg = "*Completely* remove all installed models in '{}'"
else:
msg = "The local model folder '{}' does not exist. Nothing to do."
msg = msg.format(path)
print(msg)
return
else:
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
path = utils.get_package_dir(model)
if os.path.exists(utils.get_package_cache_dir(model)):
cache = utils.get_package_cache_dir(model)
msg = "Remove '{}/'"
# Check that the model is installed.
utils.check_model_installed(model)
if utils.yes_or_no(msg, path, yes=True):
# Remove package installation dir
shutil.rmtree(path)
# Remove package config dir as well without ask
path = utils.get_package_config_dir(model)
if os.path.exists(path):
shutil.rmtree(path)
# Ask if remove cached files
if cache is not None and utils.yes_or_no(
"Remove cache '{}/' as well", cache, yes=False
):
shutil.rmtree(cache)
archive = utils.get_package_archive_dir(model)
if os.path.exists(archive):
shutil.rmtree(archive)
else:
if model is None and not args.quiet:
utils.print_next_step("remove")
def list_model_commands(args):
""" List the commands supported by this model."""
model = args.model
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
logger = logging.getLogger(__name__)
logger.info("List available commands of '{}'".format(model))
# Check that the model is installed.
utils.check_model_installed(model)
entry = utils.load_description(model)
commands = entry['commands']
if args.name_only:
print('\n'.join(list(commands)))
return
msg = "The '{}' model "
meta = entry['meta']
if 'title' not in meta:
title = None
else:
title = utils.lower_first_letter(utils.dropdot(meta['title']))
msg += "({}) "
msg += "supports the following commands:"
msg = msg.format(model, title)
msg = textwrap.fill(msg, width=75)
print(msg)
for cmd in commands:
utils.print_model_cmd_help(entry, cmd)
# Update bash completion list.
utils.update_command_completion(set(commands))
# Suggest next step.
if not args.quiet:
utils.print_next_step('commands', description=entry, model=model)
def download_model(args):
"""Download the large pre-built model."""
# TODO: Will this be a url in the DESCRIPTION file or will it be a
# download.sh script. Which ever (maybe either), if it is present
# then this command is available and will download the required
# file, perhaps from the actual source of the model.
model = args.model
# Check that the model is installed.
utils.check_model_installed(model)
if not args.quiet:
utils.print_next_step('download', model=model)
def dispatch(args):
"""Dispatch other commands to the appropriate model provided script."""
cmd = args.cmd
model = args.model
path = MLINIT + model
param = " ".join(args.param)
# Check that the model is installed.
utils.check_model_installed(model)
desc = utils.load_description(model)
# Obtain the specified script file.
script = desc["commands"][cmd]["script"].split(" ")[0] + " " + param
# Determine the interpreter to use
#
# .R => Rscript; .py => python, etc.
(root, ext) = os.path.splitext(script)
ext = ext.strip()
if ext == ".R":
interpreter = "Rscript"
elif ext == ".py":
interpreter = "python"
else:
msg = "Could not determine an interpreter for extension '{}'".format(
ext)
print(msg, file=sys.stderr)
sys.exit()
command = "{} {}".format(interpreter, script)
if args.debug:
print(DEBUG + "(cd " + path + "; " + command + ")")
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
if proc.returncode != 0:
print("An error was encountered:\n")
print(errors.decode("utf-8"))
def download_model(args):
"""Download the large pre-built model."""
# TODO: Will this be a url in the DESCRIPTION file or will it be a
# download.sh script. Which ever (maybe either), if it is present
# then this command is available and will download the required
# file, perhaps from the actual source of the model.
model = args.model
# Check that the model is installed.
utils.check_model_installed(model)
if not args.quiet:
msg = "Model information is available:\n\n $ {} readme {}\n"
msg = msg.format(CMD, model)
print(msg)
def readme(args):
"""Display the model's README information."""
# Setup.
model = args.model
path = MLINIT + model
readme = os.path.join(path, README)
# Check that the model is installed.
utils.check_model_installed(model)
# Display the README.
with open(readme, 'r') as f:
print(utils.drop_newline(f.read()))
# Suggest next step.
if not args.quiet:
utils.print_next_step('readme', model=model)
def rename_model(args):
"""Rename an installed model.
OLD is the old package name and NEW the new one.
If the NEW already exists it is overwritten only if --force.
"""
old = args.old
new = args.new
utils.check_model_installed(old)
oldp = utils.get_package_dir(old)
newp = utils.get_package_dir(new)
if os.path.exists(newp):
if args.force:
shutil.rmtree(newp)
else:
raise utils.ModelInstalledException(new)
os.rename(oldp, newp)
yfile = f"{newp}/{constants.MLHUB_YAML}"
with open(yfile) as file:
ydata = yaml.load(file, Loader=yaml.FullLoader)
ydata["meta"]["name"] = new
with open(yfile, "w") as file:
yaml.dump(ydata, file, sort_keys=False)
if not args.quiet:
msg = f"Renamed '{old}' as '{new}' (now '{newp}')."
print(msg)
def list_model_commands(args):
""" List the commands supported by this model."""
model = args.model
# Check that the model is installed.
utils.check_model_installed(model)
info = utils.load_description(model)
msg = "The model '{}' ({}) supports the following commands:"
msg = msg.format(model, info['meta']['title'])
msg = textwrap.fill(msg, width=60)
print(msg + "\n")
yaml.dump(info['commands'], sys.stdout, default_flow_style=False)
# Suggest next step.
if not args.quiet:
msg = "Model dependencies are listed using:\n\n $ {} configure {}\n"
msg = msg.format(CMD, model)
print(msg)
def readme(args):
"""Display the model's README information."""
# Setup.
model = args.model
path = MLINIT + model
readme = os.path.join(path, README)
# Check that the model is installed.
utils.check_model_installed(model)
# Display the README.
with open(readme, 'r') as f:
print(f.read())
# Suggest next step.
if not args.quiet:
msg = "Model dependencies are listed using:\n\n $ {} configure {}\n"
msg = msg.format(CMD, model)
print(msg)
def dispatch(args):
"""Dispatch other commands to the appropriate model provided script."""
cmd = args.cmd
model = args.model
path = utils.get_package_dir(model)
param = " ".join(args.param)
# Get working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
if args.workding_dir == '':
args.workding_dir = None
else:
args.working_dir = utils.get_working_dir(model)
# Get conda environment name if any.
conda_env_name = utils.get_conda_env_name(model)
# Check that the model is installed and has commands.
utils.check_model_installed(model)
entry = utils.load_description(model)
if 'commands' not in entry or len(entry['commands']) == 0:
raise utils.CommandNotFoundException(cmd, model)
# Correct misspelled command if possible.
matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands']))
if matched_cmd is not None:
cmd = matched_cmd
# Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.
meta = entry['meta']
if 'display' in meta and cmd in meta['display'] and os.environ.get(
'DISPLAY', '') == '':
msg = "Graphic display is required but not available for command '{}'. Continue"
yes = utils.yes_or_no(msg, cmd, yes=False)
if not yes:
msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.
"""
sys.stdout.write(msg)
sys.exit(1)
# Obtain the default/chosen language for the package.
lang = meta["languages"]
# Deal with malformed 'languages' field
lang_opts = {"python": "py", "R": "R"}
for k in list(lang_opts):
if lang in k:
lang = lang_opts[k]
break
# Obtain the specified script file.
script = cmd + "." + lang
logger = logging.getLogger(__name__)
logger.debug("Execute the script: " + os.path.join(path, script))
if cmd not in list(entry['commands']) or not os.path.exists(
os.path.join(path, script)):
raise utils.CommandNotFoundException(cmd, model)
# Determine the interpreter to use
#
# .R => Rscript; .py => python, etc.
interpreter = utils.interpreter(script)
# Change working dir if needed
if args.workding_dir is not None:
script = os.path.join(path, script)
path = args.workding_dir
# _MLHUB_CMD_CWD: a environment variable indicates current working
# directory where command `ml xxx` is invoked.
# _MLHUB_MODEL_NAME: env variable indicates the name of the model.
#
# The above two env vars can be obtained by helper function, such
# as utils.get_cmd_cwd(). And model package developer should be
# use the helper function instead of the env vars directly.
command = "export _MLHUB_CMD_CWD='{}'; export _MLHUB_MODEL_NAME='{}'; {} {} {}".format(
os.getcwd(), model, interpreter, script, param)
# Run script inside conda environment if specified
if conda_env_name is not None:
command = 'bash -c "source activate {}; {}"'.format(
conda_env_name, command)
logger.debug("(cd " + path + "; " + command + ")")
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
missing_r_dep = False
if proc.returncode != 0:
errors = errors.decode("utf-8")
# Check if it is Python dependency unsatisfied
dep_required = re.compile(
r"ModuleNotFoundError: No module named '(.*)'").search(errors)
# Check if R dependency unsatisified
if dep_required is None:
dep_required = re.compile(
r"there is no package called ‘(.*)’").search(errors)
if dep_required is not None:
missing_r_dep = True
# Check if required data resource not found
data_required = re.compile(
r"mlhub.utils.DataResourceNotFoundException").search(errors)
if dep_required is not None: # Dependency unsatisfied
dep_required = dep_required.group(1)
logger.error("Dependency unsatisfied: {}\n{}".format(
dep_required, errors))
raise utils.LackDependencyException(dep_required, missing_r_dep,
model)
elif data_required is not None: # Data not found
raise utils.DataResourceNotFoundException()
else: # Other unknown errors
print("An error was encountered:\n")
print(errors)
else:
# Suggest next step
if not args.quiet:
utils.print_next_step(cmd, description=entry, model=model)
def configure_model(args):
"""Ensure the user's environment is configured."""
# TODO: Add support for additional configuration if any except those
# specified in MLHUB.yaml.
# TODO: When fail, print out the failed dep, as well as installed
# deps and non-installed deps.
# TODO: Add support for specifying packages version.
# TODO: Add more informative messages for different kinds of
# dependencies.
# Other ideas for configuration
#
# 1 Construct mlhub container (from Ubuntu) with known starting point
#
# 2 Assume the user is on a DSVM with free Azure account to test out.
#
# 3 Read dependencies: and language: and then install as required:
#
# 4 Assume model packager provides a configure.R script. This is a
# override and no other configuration happens if this is
# supplied. Alternatively this is viewed as a cop-out prividing
# no support from mlhub for the model packager. The preference
# would be to use the dependencies: tag to list the required R
# or python packages.
#
# So the meta-code might be
#
# if file.exists(configure.XX):
# XX configure.XX
# else if language: == "Rscript":
# packages <- dependencies:
# install <- packages[!(packages %in% installed.packages()[,"Package"])]
# if(length(new.packages)) install.packages(install)
# else if language: == "python":
# packages = dependencies:
# cat pacakges > requirements.txt
# pip install -r requirements.txt
#
if not args.model:
# Configure MLHUB per se.
# Includes bash completion and system pre-requisites
if distro.id() in ['debian', 'ubuntu']:
path = os.path.dirname(__file__)
command = '/bin/bash {}'.format(
os.path.join('scripts', 'dep', 'mlhub.sh'))
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
if proc.returncode != 0:
errors = errors.decode("utf-8")
print("\nAn error was encountered:\n")
print(errors)
raise utils.ConfigureFailedException()
return
# Model package configuration.
model = args.model
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
# Setup.
pkg_dir = utils.get_package_dir(model)
# Check if the model package is installed.
utils.check_model_installed(model)
# Install dependencies specified in MLHUB.yaml
entry = utils.load_description(model)
depspec = None
if 'dependencies' in entry:
depspec = entry['dependencies']
elif 'dependencies' in entry['meta']:
depspec = entry['meta']['dependencies']
if depspec is not None:
for spec in utils.flatten_mlhubyaml_deps(depspec):
category = spec[0][-1]
deplist = spec[1]
# Category include:
# ------------------------------------------------------------------------------
# category | action
# -----------------|------------------------------------------------------------
# None | install package according to entry['meta']['languages']
# | if R, install.packages(xxx) from cran;
# | if Python, pip install xxx
# -----------------|------------------------------------------------------------
# system | apt-get install
# sh | apt-get install
# -----------------|------------------------------------------------------------
# r | install.packages(xxx) from cran, version can be specified
# cran | install.packages(xxx) from cran, version can be specified
# cran-2018-12-01 | install cran snapshot on 2018-12-01
# github | devtools::install_github from github
# -----------------|------------------------------------------------------------
# python | apt-get install python-xxx
# python3 | apt-get install python3-xxx
# pip | pip install
# pip3 | pip3 install
# conda | conda install
# -----------------|------------------------------------------------------------
# files | download files
# -----------------|------------------------------------------------------------
# ----- Determine deps by language -----
if category is None:
lang = entry['meta']['languages'].lower()
if lang == 'r':
utils.install_r_deps(deplist, model, source='cran')
elif 'python'.startswith(lang):
utils.install_python_deps(deplist, model, source='pip')
# ----- System deps -----
elif category == 'system' or 'shell'.startswith(category):
utils.install_system_deps(deplist)
# ----- R deps -----
elif category == 'r':
utils.install_r_deps(deplist, model, source='cran')
elif category == 'cran' or category == 'github' or category.startswith(
'cran-'):
utils.install_r_deps(deplist, model, source=category)
# ----- Python deps -----
elif category.startswith('python') or category.startswith(
'pip') or category == 'conda':
utils.install_python_deps(deplist, model, source=category)
# ----- Files -----
elif 'files'.startswith(category):
utils.install_file_deps(deplist, model)
# Run additional configure script if any.
conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True
conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf
conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf
if not conf:
if depspec is not None:
msg = ("No configuration script provided for this model. "
"The following dependencies are required:\n")
print(msg)
print(yaml.dump(depspec))
else:
print("No configuration provided (maybe none is required).")
# Update working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
# Suggest next step.
if not args.quiet:
utils.print_next_step('configure', model=model)
def configure_model(args):
"""Ensure the user's environment is configured."""
# TODO: Install packages natively for those listed in
# dependencies. Then if there is also a configure.sh, then run
# that for additoinal setup.
# Other ideas re cofiguration
#
# 1 Construct mlhub container (from Ubuntu) with known starting point
#
# 2 Assume the user is on a DSVM with free Azure account to test out.
#
# 3 Read dependencies: and language: and then install as required:
#
# 4 Assume model packager provides a configure.R script. This is a
# override and no other configuration happens if this is
# supplied. Alternatively this is viewed as a cop-out prividing
# no support from mlhub for the model packager. The preference
# would be to use the dependencies: tag to list the required R
# or python packages.
#
# So the meta-code might be
#
# if file.exists(configure.XX):
# XX configure.XX
# else if language: == "Rscript":
# packages <- dependencies:
# install <- packages[!(packages %in% installed.packages()[,"Package"])]
# if(length(new.packages)) install.packages(install)
# else if language: == "python":
# packages = dependencies:
# cat pacakges > requirements.txt
# pip install -r requirements.txt
#
# Setup.
model = args.model
path = MLINIT + model
# Check that the model is installed.
utils.check_model_installed(model)
# If there is a configure script and this is a Ubuntu host, then
# run the configure script.
conf = os.path.join(path, "configure.sh")
if platform.dist()[0] in set(['debian', 'Ubuntu'
]) and os.path.exists(conf):
command = "bash configure.sh"
if not args.quiet:
msg = "Configuration will take place using '{}'.\n"
msg = msg.format(conf)
print(msg)
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
if proc.returncode != 0:
print("An error was encountered:\n")
print(errors.decode("utf-8"))
else:
# For now simply list the declared dependencies for the user to
# make sure they have it all installed.
if not args.quiet:
msg = """
Configuration is yet to be automated. The following dependencies are required:
"""
print(msg)
info = utils.load_description(model)
msg = info["meta"]["dependencies"]
print(" ====> \033[31m" + msg + "\033[0m")
# Suggest next step.
if not args.quiet:
msg = "\nOnce configured run the demonstration:\n\n $ {} demo {}\n"
msg = msg.format(CMD, model)
print(msg)
def dispatch(args):
"""Dispatch other commands to the appropriate model provided script."""
cmd = args.cmd
model = args.model
path = utils.get_package_dir(model)
param = " ".join(args.param)
# Get working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
if args.workding_dir == '':
args.workding_dir = None
else:
args.working_dir = utils.get_working_dir(model)
# Get conda environment name if any.
conda_env_name = utils.get_conda_env_name(model)
# Check that the model is installed and has commands.
utils.check_model_installed(model)
entry = utils.load_description(model)
if 'commands' not in entry or len(entry['commands']) == 0:
raise utils.CommandNotFoundException(cmd, model)
# Correct misspelled command if possible.
matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands']))
if matched_cmd is not None:
cmd = matched_cmd
# Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.
meta = entry['meta']
if 'display' in meta and cmd in meta['display'] and os.environ.get(
'DISPLAY', '') == '':
msg = "Graphic display is required but not available for command '{}'. Continue"
yes = utils.yes_or_no(msg, cmd, yes=False)
if not yes:
msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.
"""
sys.stdout.write(msg)
sys.exit(1)
# Obtain the default/chosen language for the package.
lang = meta["languages"]
# Deal with malformed 'languages' field
lang_opts = {"python": "py", "R": "R"}
for k in list(lang_opts):
if lang in k:
lang = lang_opts[k]
break
# Obtain the specified script file.
script = cmd + "." + lang
logger = logging.getLogger(__name__)
logger.debug("Execute the script: " + os.path.join(path, script))
if cmd not in list(entry['commands']) or not os.path.exists(
os.path.join(path, script)):
raise utils.CommandNotFoundException(cmd, model)
# Determine the interpreter to use
#
# .R => Rscript; .py => python, etc.
interpreter = utils.interpreter(script)
# Change working dir if needed
if args.workding_dir is not None:
script = os.path.join(path, script)
path = args.workding_dir
# Handle python environment
python_pkg_bin = None
python_pkg_path = None
if script.endswith('py'):
python_pkg_base = os.path.sep.join(
[utils.get_package_dir(model), '.python'])
python_pkg_path = python_pkg_base + site.USER_SITE
python_pkg_bin = python_pkg_base + site.USER_BASE + '/bin'
# TODO: Make sure to document:
# $ sudo apt-get install -y python3-pip
# $ /usr/bin/pip3 install mlhub
# Since in DSVM, the default pip is conda's pip, so if we stick to
# use system's command, then the installation of MLHub itself should
# be completed via system's pip, otherwise, MLHub will not work.
if sys.executable != SYS_PYTHON_CMD:
python_pkg_path = python_pkg_base + site.getsitepackages()[0]
python_pkg_bin = python_pkg_base + site.PREFIXES[0] + '/bin'
if utils.get_sys_python_pkg_usage(model):
utils.print_on_stderr(MSG_INCOMPATIBLE_PYTHON_ENV, model)
# _MLHUB_CMD_CWD: a environment variable indicates current working
# directory where command `ml xxx` is invoked.
# _MLHUB_MODEL_NAME: env variable indicates the name of the model.
#
# The above two env vars can be obtained by helper function, such
# as utils.get_cmd_cwd(). And model package developer should be
# use the helper function instead of the env vars directly.
env_var = "export _MLHUB_CMD_CWD='{}'; ".format(os.getcwd())
env_var += "export _MLHUB_MODEL_NAME='{}'; ".format(model)
env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable)
env_var += "export PYTHONPATH='{}'; ".format(
python_pkg_path) if python_pkg_path else ""
env_var += "export PATH=\"{}:$PATH\"; ".format(
python_pkg_bin) if python_pkg_bin else ""
command = "{}{} {} {}".format(env_var, interpreter, script, param)
# Run script inside conda environment if specified
if conda_env_name is not None:
command = '{} -c "source activate {}; {}"'.format(
BASH_CMD, conda_env_name, command)
logger.debug("(cd " + path + "; " + command + ")")
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
missing_r_dep = False
if proc.returncode != 0:
errors = errors.decode("utf-8")
# Check if it is Python dependency unsatisfied
dep_required = re.compile(
r"ModuleNotFoundError: No module named '(.*)'").search(errors)
# Check if R dependency unsatisified
if dep_required is None:
dep_required = re.compile(
r"there is no package called ‘(.*)’").search(errors)
if dep_required is not None:
missing_r_dep = True
# Check if required data resource not found
data_required = re.compile(
r"mlhub.utils.DataResourceNotFoundException").search(errors)
if dep_required is not None: # Dependency unsatisfied
dep_required = dep_required.group(1)
logger.error("Dependency unsatisfied: {}\n{}".format(
dep_required, errors))
raise utils.LackDependencyException(dep_required, missing_r_dep,
model)
elif data_required is not None: # Data not found
raise utils.DataResourceNotFoundException()
else: # Other unknown errors
print("An error was encountered:\n")
print(errors)
else:
# Suggest next step
if not args.quiet:
utils.print_next_step(cmd, description=entry, model=model)
def dispatch(args):
"""Dispatch other commands to the appropriate model provided script."""
cmd = args.cmd
model = args.model
path = MLINIT + model
param = " ".join(args.param)
# Check that the model is installed.
utils.check_model_installed(model)
desc = utils.load_description(model)
# Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.
if 'display' in desc['meta'] and cmd in desc['meta']['display'] and os.environ.get('DISPLAY', '') == '':
msg = "Graphic display is required but not available for command '{}'. Continue [y/N]? "
msg = msg.format(cmd)
sys.stdout.write(msg)
choice = input().lower()
if choice != 'y':
msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.
"""
sys.stdout.write(msg)
sys.exit(1)
# Obtain the default/chosen language for the package.
lang = desc["meta"]["languages"]
# Deal with malformed 'languages' field
lang_opts = {"python": "py", "R": "R"}
for k in list(lang_opts):
if lang in k:
lang = lang_opts[k]
break
# Obtain the specified script file.
script = cmd + "." + lang
if args.debug:
print(DEBUG + "execute the script: " + os.path.join(path, script))
if cmd not in list(desc['commands']) or not os.path.exists(os.path.join(path, script)):
msg = """{}The command '{}' was not found for this model.
Try using 'commands' to list all supported commands:
$ {} commands {}
""".format(APPX, cmd, CMD, model)
print(msg, file=sys.stderr)
sys.exit(1)
# Determine the interpreter to use
#
# .R => Rscript; .py => python, etc.
interpreter = utils.interpreter(script)
command = "export CMD_CWD='{}'; {} {} {}".format(os.getcwd(), interpreter, script, param)
if args.debug:
print(DEBUG + "(cd " + path + "; " + command + ")")
proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE)
output, errors = proc.communicate()
if proc.returncode != 0:
print("An error was encountered:\n")
print(errors.decode("utf-8"))
else:
# Suggest next step
if not args.quiet:
utils.print_next_step(cmd, description=desc, model=model)
def configure_model(args):
"""Ensure the user's environment is configured."""
# TODO: Install packages natively for those listed in
# dependencies. Then if there is also a configure.sh, then run
# that for additoinal setup.
# Other ideas re cofiguration
#
# 1 Construct mlhub container (from Ubuntu) with known starting point
#
# 2 Assume the user is on a DSVM with free Azure account to test out.
#
# 3 Read dependencies: and language: and then install as required:
#
# 4 Assume model packager provides a configure.R script. This is a
# override and no other configuration happens if this is
# supplied. Alternatively this is viewed as a cop-out prividing
# no support from mlhub for the model packager. The preference
# would be to use the dependencies: tag to list the required R
# or python packages.
#
# So the meta-code might be
#
# if file.exists(configure.XX):
# XX configure.XX
# else if language: == "Rscript":
# packages <- dependencies:
# install <- packages[!(packages %in% installed.packages()[,"Package"])]
# if(length(new.packages)) install.packages(install)
# else if language: == "python":
# packages = dependencies:
# cat pacakges > requirements.txt
# pip install -r requirements.txt
#
if not args.model:
# Configure ml. Currently only bash completion.
import platform
sys_version = platform.uname().version.lower()
if 'debian' in sys_version or 'ubuntu' in sys_version:
path = os.path.dirname(__file__)
commands = [
'sudo cp {} /etc/bash_completion.d'.format(COMPLETION_SCRIPT),
'ml available > /dev/null',
'ml installed > /dev/null', ]
for cmd in commands:
print('Executing: ', cmd)
subprocess.run(cmd, shell=True, cwd=path, stderr=subprocess.PIPE)
print("\nLast step to make tab completion take effect: \n\n $ source /etc/bash_completion.d/ml.bash")
return
# Setup.
model = args.model
path = MLINIT + model
# Check that the model is installed.
utils.check_model_installed(model)
# If there are any configure scripts then run them, else print the
# list of supplied dependencies if any. Note that Python's 'or' is
# lazy evaluation.
conf = utils.configure(path, "configure.sh", args.quiet)
conf = utils.configure(path, "configure.R", args.quiet) or conf
conf = utils.configure(path, "configure.py", args.quiet) or conf
if not conf:
try:
info = utils.load_description(model)
deps = info["meta"]["dependencies"]
if not args.quiet:
msg = "No configuration script provided for this model. "
msg = msg + "The following dependencies are required:\n"
print(msg)
print(" ====> \033[31m" + deps + "\033[0m")
except:
print("No configuration provided (maybe none is required).")
# Suggest next step.
if not args.quiet:
utils.print_next_step('configure', model=model)
def list_model_commands(args):
""" List the commands supported by this model."""
# Setup.
model = args.model
# Check that the model is installed.
utils.check_model_installed(model)
info = utils.load_description(model)
if args.name_only:
print('\n'.join(list(info['commands'])))
return
msg = "The model '{}' "
if 'title' not in info['meta']:
title = None
else:
title = utils.lower_first_letter(utils.dropdot(info['meta']['title']))
msg += "({}) "
msg += "supports the following commands:"
msg = msg.format(model, title)
msg = textwrap.fill(msg, width=75)
print(msg)
for c in info['commands']:
print("\n $ {} {} {}".format(CMD, c, model))
c_meta = info['commands'][c]
if type(c_meta) is str:
print(" " + c_meta)
else:
# Handle malformed DESCRIPTION.yaml like
# --
# commands:
# print:
# description: print a textual summary of the model
# score:
# equired: the name of a CSV file containing a header and 6 columns
# description: apply the model to a supplied dataset
desc = c_meta.get('description', None)
if desc is not None:
print(" " + desc)
c_meta = {k:c_meta[k] for k in c_meta if k != 'description'}
if len(c_meta) > 0:
msg = yaml.dump(c_meta, default_flow_style=False)
msg = msg.split('\n')
msg = [" " + ele for ele in msg]
print('\n'.join(msg), end='')
# Update available commands for the model for fast bash tab completion.
utils.update_completion_list(COMPLETION_COMMANDS, set(info['commands']))
# Suggest next step.
if not args.quiet:
utils.print_next_step('commands', description=info, model=model)
def configure_model(args):
"""Ensure the user's environment is configured."""
# TODO: Add support for additional configuration if any except those
# specified in MLHUB.yaml.
# TODO: When fail, print out the failed dep, as well as installed
# deps and non-installed deps.
# TODO: Add support for specifying packages version.
# TODO: Add more informative messages for different kinds of
# dependencies.
# Other ideas for configuration
#
# 1 Construct mlhub container (from Ubuntu) with known starting point
#
# 2 Assume the user is on a DSVM with free Azure account to test out.
#
# 3 Read dependencies: and language: and then install as required:
#
# 4 Assume model packager provides a configure.R script. This is a
# override and no other configuration happens if this is
# supplied. Alternatively this is viewed as a cop-out providing
# no support from mlhub for the model packager. The preference
# would be to use the dependencies: tag to list the required R
# or python packages.
#
# So the meta-code might be
#
# if file.exists(configure.XX):
# XX configure.XX
# else if language: == "Rscript":
# packages <- dependencies:
# install <- packages[!(packages %in% installed.packages()[,"Package"])]
# if(length(new.packages)) install.packages(install)
# else if language: == "python":
# packages = dependencies:
# cat packages > requirements.txt
# pip install -r requirements.txt
#
YES = args.y | args.yes
# Avoid 403 errors which result when the header identifies itself
# as python urllib or is empty and thus the web site assumes it is
# a robot. We are not a robot but a user downloading a file. This
# will ensure gitlab is okay with retrieving from a URL by adding
# a header rather than no header. TODO move to using Requests.
opener = urllib.request.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
urllib.request.install_opener(opener)
if not args.model:
# Configure MLHUB per se.
# Includes bash completion and system pre-requisites
if distro.id() in ["debian", "ubuntu"]:
path = os.path.dirname(__file__)
env_var = "export _MLHUB_OPTION_YES='y'; " if YES else ""
env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable)
script = os.path.join("scripts", "dep", "mlhub.sh")
command = "{}{} {}".format(env_var, BASH_CMD, script)
proc = subprocess.Popen(
command, shell=True, cwd=path, stderr=subprocess.PIPE
)
output, errors = proc.communicate()
if proc.returncode != 0:
raise utils.ConfigureFailedException(errors.decode("utf-8"))
return
# Model package configuration.
model = args.model
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
# Setup.
pkg_dir = utils.get_package_dir(model)
# Check if the model package is installed.
utils.check_model_installed(model)
# Install dependencies specified in MLHUB.yaml
entry = utils.load_description(model)
depspec = None
if "dependencies" in entry:
depspec = entry["dependencies"]
elif "dependencies" in entry["meta"]:
depspec = entry["meta"]["dependencies"]
if depspec is not None:
for spec in utils.flatten_mlhubyaml_deps(depspec):
category = spec[0][-1]
deplist = spec[1]
# Category include:
# ------------------------------------------------------------------------------
# category | action
# -----------------|------------------------------------------------------------
# None | install package according to entry['meta']['languages']
# | if R, install.packages(xxx) from cran;
# | if Python, pip install xxx
# -----------------|------------------------------------------------------------
# system | apt-get install
# sh | apt-get install
# -----------------|------------------------------------------------------------
# r | install.packages(xxx) from cran, version can be specified
# cran | install.packages(xxx) from cran, version can be specified
# cran-2018-12-01 | install cran snapshot on 2018-12-01
# github | devtools::install_github from github
# -----------------|------------------------------------------------------------
# python | apt-get install python-xxx
# python3 | apt-get install python3-xxx
# pip | pip install
# pip3 | pip3 install
# conda | conda install
# -----------------|------------------------------------------------------------
# files | download files
# -----------------|------------------------------------------------------------
# ----- Determine deps by language -----
if category is None:
lang = entry["meta"]["languages"].lower()
if lang == "r":
utils.install_r_deps(
deplist, model, source="cran", yes=YES
)
elif "python".startswith(lang):
utils.install_python_deps(
deplist, model, source="pip", yes=YES
)
# ----- System deps -----
elif category == "system" or "shell".startswith(category):
utils.install_system_deps(deplist, yes=YES)
# ----- R deps -----
elif category == "r":
utils.install_r_deps(deplist, model, source="cran", yes=YES)
elif (
category == "cran"
or category == "github"
or category.startswith("cran-")
):
utils.install_r_deps(deplist, model, source=category, yes=YES)
# ----- Python deps -----
elif (
category.startswith("python")
or category.startswith("pip")
or category == "conda"
):
utils.install_python_deps(
deplist, model, source=category, yes=YES
)
# ----- Files -----
elif "files".startswith(category):
utils.install_file_deps(deplist, model, key=args.i, yes=YES)
# Run additional configure script if any.
conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True
conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf
conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf
if not conf:
if depspec is not None:
msg = (
"No configuration script provided for this model. "
"The following dependencies are required:\n"
)
print(msg)
print(yaml.dump(depspec))
else:
print("No configuration provided (maybe none is required).")
# Update working dir if any.
if args.working_dir is not None:
utils.update_working_dir(model, args.working_dir)
# Suggest next step.
if not args.quiet:
utils.print_next_step("configure", model=model)
