def configure_model(args):
"""Ensure the user's environment is configured."""
# TODO: Add support for additional configuration if any except those
# specified in MLHUB.yaml.
# TODO: When fail, print out the failed dep, as well as installed
# deps and non-installed deps.
# TODO: Add support for specifying packages version.
# TODO: Add more informative messages for different kinds of
# dependencies.
# Other ideas for configuration
#
# 1 Construct mlhub container (from Ubuntu) with known starting point
#
# 2 Assume the user is on a DSVM with free Azure account to test out.
#
# 3 Read dependencies: and language: and then install as required:
#
# 4 Assume model packager provides a configure.R script. This is a
# override and no other configuration happens if this is
# supplied. Alternatively this is viewed as a cop-out prividing
# no support from mlhub for the model packager. The preference
# would be to use the dependencies: tag to list the required R
# or python packages.
#
# So the meta-code might be
#
# if file.exists(configure.XX):
# XX configure.XX
# else if language: == "Rscript":
# packages <- dependencies:
# install <- packages[!(packages %in% installed.packages()[,"Package"])]
# if(length(new.packages)) install.packages(install)
# else if language: == "python":
# packages = dependencies:
# cat pacakges > requirements.txt
# pip install -r requirements.txt
#
if not args.model:
# Configure MLHUB per se.
# Includes bash completion and system pre-requisites
if distro.id() in ['debian', 'ubuntu']:
path = os.path.dirname(__file__)
command = '/bin/bash {}'.format(
os.path.join('scripts', 'dep', 'mlhub.sh'))
proc = subprocess.Popen(command,
shell=True,
cwd=path,
stderr=subprocess.PIPE)
output, errors = proc.communicate()
if proc.returncode != 0:
errors = errors.decode("utf-8")
print("\nAn error was encountered:\n")
print(errors)
raise utils.ConfigureFailedException()
return
# Model package configuration.
model = args.model
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
# Setup.
pkg_dir = utils.get_package_dir(model)
# Check if the model package is installed.
utils.check_model_installed(model)
# Install dependencies specified in MLHUB.yaml
entry = utils.load_description(model)
depspec = None
if 'dependencies' in entry:
depspec = entry['dependencies']
elif 'dependencies' in entry['meta']:
depspec = entry['meta']['dependencies']
if depspec is not None:
for spec in utils.flatten_mlhubyaml_deps(depspec):
category = spec[0][-1]
deplist = spec[1]
# Category include:
# ------------------------------------------------------------------------------
# category | action
# -----------------|------------------------------------------------------------
# None | install package according to entry['meta']['languages']
# | if R, install.packages(xxx) from cran;
# | if Python, pip install xxx
# -----------------|------------------------------------------------------------
# system | apt-get install
# sh | apt-get install
# -----------------|------------------------------------------------------------
# r | install.packages(xxx) from cran, version can be specified
# cran | install.packages(xxx) from cran, version can be specified
# cran-2018-12-01 | install cran snapshot on 2018-12-01
# github | devtools::install_github from github
# -----------------|------------------------------------------------------------
# python | apt-get install python-xxx
# python3 | apt-get install python3-xxx
# pip | pip install
# pip3 | pip3 install
# conda | conda install
# -----------------|------------------------------------------------------------
# files | download files
# -----------------|------------------------------------------------------------
# ----- Determine deps by language -----
if category is None:
lang = entry['meta']['languages'].lower()
if lang == 'r':
utils.install_r_deps(deplist, model, source='cran')
elif 'python'.startswith(lang):
utils.install_python_deps(deplist, model, source='pip')
# ----- System deps -----
elif category == 'system' or 'shell'.startswith(category):
utils.install_system_deps(deplist)
# ----- R deps -----
elif category == 'r':
utils.install_r_deps(deplist, model, source='cran')
elif category == 'cran' or category == 'github' or category.startswith(
'cran-'):
utils.install_r_deps(deplist, model, source=category)
# ----- Python deps -----
elif category.startswith('python') or category.startswith(
'pip') or category == 'conda':
utils.install_python_deps(deplist, model, source=category)
# ----- Files -----
elif 'files'.startswith(category):
utils.install_file_deps(deplist, model)
# Run additional configure script if any.
conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True
conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf
conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf
if not conf:
if depspec is not None:
msg = ("No configuration script provided for this model. "
"The following dependencies are required:\n")
print(msg)
print(yaml.dump(depspec))
else:
print("No configuration provided (maybe none is required).")
# Update working dir if any.
if args.workding_dir is not None:
utils.update_working_dir(model, args.workding_dir)
# Suggest next step.
if not args.quiet:
utils.print_next_step('configure', model=model)
def configure_model(args):
"""Ensure the user's environment is configured."""
# TODO: Add support for additional configuration if any except those
# specified in MLHUB.yaml.
# TODO: When fail, print out the failed dep, as well as installed
# deps and non-installed deps.
# TODO: Add support for specifying packages version.
# TODO: Add more informative messages for different kinds of
# dependencies.
# Other ideas for configuration
#
# 1 Construct mlhub container (from Ubuntu) with known starting point
#
# 2 Assume the user is on a DSVM with free Azure account to test out.
#
# 3 Read dependencies: and language: and then install as required:
#
# 4 Assume model packager provides a configure.R script. This is a
# override and no other configuration happens if this is
# supplied. Alternatively this is viewed as a cop-out providing
# no support from mlhub for the model packager. The preference
# would be to use the dependencies: tag to list the required R
# or python packages.
#
# So the meta-code might be
#
# if file.exists(configure.XX):
# XX configure.XX
# else if language: == "Rscript":
# packages <- dependencies:
# install <- packages[!(packages %in% installed.packages()[,"Package"])]
# if(length(new.packages)) install.packages(install)
# else if language: == "python":
# packages = dependencies:
# cat packages > requirements.txt
# pip install -r requirements.txt
#
YES = args.y | args.yes
# Avoid 403 errors which result when the header identifies itself
# as python urllib or is empty and thus the web site assumes it is
# a robot. We are not a robot but a user downloading a file. This
# will ensure gitlab is okay with retrieving from a URL by adding
# a header rather than no header. TODO move to using Requests.
opener = urllib.request.build_opener()
opener.addheaders = [('User-agent', 'Mozilla/5.0')]
urllib.request.install_opener(opener)
if not args.model:
# Configure MLHUB per se.
# Includes bash completion and system pre-requisites
if distro.id() in ["debian", "ubuntu"]:
path = os.path.dirname(__file__)
env_var = "export _MLHUB_OPTION_YES='y'; " if YES else ""
env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable)
script = os.path.join("scripts", "dep", "mlhub.sh")
command = "{}{} {}".format(env_var, BASH_CMD, script)
proc = subprocess.Popen(
command, shell=True, cwd=path, stderr=subprocess.PIPE
)
output, errors = proc.communicate()
if proc.returncode != 0:
raise utils.ConfigureFailedException(errors.decode("utf-8"))
return
# Model package configuration.
model = args.model
# Correct model name if possible.
matched_model = utils.get_misspelled_pkg(model)
if matched_model is not None:
model = matched_model
# Setup.
pkg_dir = utils.get_package_dir(model)
# Check if the model package is installed.
utils.check_model_installed(model)
# Install dependencies specified in MLHUB.yaml
entry = utils.load_description(model)
depspec = None
if "dependencies" in entry:
depspec = entry["dependencies"]
elif "dependencies" in entry["meta"]:
depspec = entry["meta"]["dependencies"]
if depspec is not None:
for spec in utils.flatten_mlhubyaml_deps(depspec):
category = spec[0][-1]
deplist = spec[1]
# Category include:
# ------------------------------------------------------------------------------
# category | action
# -----------------|------------------------------------------------------------
# None | install package according to entry['meta']['languages']
# | if R, install.packages(xxx) from cran;
# | if Python, pip install xxx
# -----------------|------------------------------------------------------------
# system | apt-get install
# sh | apt-get install
# -----------------|------------------------------------------------------------
# r | install.packages(xxx) from cran, version can be specified
# cran | install.packages(xxx) from cran, version can be specified
# cran-2018-12-01 | install cran snapshot on 2018-12-01
# github | devtools::install_github from github
# -----------------|------------------------------------------------------------
# python | apt-get install python-xxx
# python3 | apt-get install python3-xxx
# pip | pip install
# pip3 | pip3 install
# conda | conda install
# -----------------|------------------------------------------------------------
# files | download files
# -----------------|------------------------------------------------------------
# ----- Determine deps by language -----
if category is None:
lang = entry["meta"]["languages"].lower()
if lang == "r":
utils.install_r_deps(
deplist, model, source="cran", yes=YES
)
elif "python".startswith(lang):
utils.install_python_deps(
deplist, model, source="pip", yes=YES
)
# ----- System deps -----
elif category == "system" or "shell".startswith(category):
utils.install_system_deps(deplist, yes=YES)
# ----- R deps -----
elif category == "r":
utils.install_r_deps(deplist, model, source="cran", yes=YES)
elif (
category == "cran"
or category == "github"
or category.startswith("cran-")
):
utils.install_r_deps(deplist, model, source=category, yes=YES)
# ----- Python deps -----
elif (
category.startswith("python")
or category.startswith("pip")
or category == "conda"
):
utils.install_python_deps(
deplist, model, source=category, yes=YES
)
# ----- Files -----
elif "files".startswith(category):
utils.install_file_deps(deplist, model, key=args.i, yes=YES)
# Run additional configure script if any.
conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True
conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf
conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf
if not conf:
if depspec is not None:
msg = (
"No configuration script provided for this model. "
"The following dependencies are required:\n"
)
print(msg)
print(yaml.dump(depspec))
else:
print("No configuration provided (maybe none is required).")
# Update working dir if any.
if args.working_dir is not None:
utils.update_working_dir(model, args.working_dir)
# Suggest next step.
if not args.quiet:
utils.print_next_step("configure", model=model)