def configure_model(args): """Ensure the user's environment is configured.""" # TODO: Add support for additional configuration if any except those # specified in MLHUB.yaml. # TODO: When fail, print out the failed dep, as well as installed # deps and non-installed deps. # TODO: Add support for specifying packages version. # TODO: Add more informative messages for different kinds of # dependencies. # Other ideas for configuration # # 1 Construct mlhub container (from Ubuntu) with known starting point # # 2 Assume the user is on a DSVM with free Azure account to test out. # # 3 Read dependencies: and language: and then install as required: # # 4 Assume model packager provides a configure.R script. This is a # override and no other configuration happens if this is # supplied. Alternatively this is viewed as a cop-out prividing # no support from mlhub for the model packager. The preference # would be to use the dependencies: tag to list the required R # or python packages. # # So the meta-code might be # # if file.exists(configure.XX): # XX configure.XX # else if language: == "Rscript": # packages <- dependencies: # install <- packages[!(packages %in% installed.packages()[,"Package"])] # if(length(new.packages)) install.packages(install) # else if language: == "python": # packages = dependencies: # cat pacakges > requirements.txt # pip install -r requirements.txt # if not args.model: # Configure MLHUB per se. # Includes bash completion and system pre-requisites if distro.id() in ['debian', 'ubuntu']: path = os.path.dirname(__file__) command = '/bin/bash {}'.format( os.path.join('scripts', 'dep', 'mlhub.sh')) proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() if proc.returncode != 0: errors = errors.decode("utf-8") print("\nAn error was encountered:\n") print(errors) raise utils.ConfigureFailedException() return # Model package configuration. model = args.model # Correct model name if possible. matched_model = utils.get_misspelled_pkg(model) if matched_model is not None: model = matched_model # Setup. pkg_dir = utils.get_package_dir(model) # Check if the model package is installed. utils.check_model_installed(model) # Install dependencies specified in MLHUB.yaml entry = utils.load_description(model) depspec = None if 'dependencies' in entry: depspec = entry['dependencies'] elif 'dependencies' in entry['meta']: depspec = entry['meta']['dependencies'] if depspec is not None: for spec in utils.flatten_mlhubyaml_deps(depspec): category = spec[0][-1] deplist = spec[1] # Category include: # ------------------------------------------------------------------------------ # category | action # -----------------|------------------------------------------------------------ # None | install package according to entry['meta']['languages'] # | if R, install.packages(xxx) from cran; # | if Python, pip install xxx # -----------------|------------------------------------------------------------ # system | apt-get install # sh | apt-get install # -----------------|------------------------------------------------------------ # r | install.packages(xxx) from cran, version can be specified # cran | install.packages(xxx) from cran, version can be specified # cran-2018-12-01 | install cran snapshot on 2018-12-01 # github | devtools::install_github from github # -----------------|------------------------------------------------------------ # python | apt-get install python-xxx # python3 | apt-get install python3-xxx # pip | pip install # pip3 | pip3 install # conda | conda install # -----------------|------------------------------------------------------------ # files | download files # -----------------|------------------------------------------------------------ # ----- Determine deps by language ----- if category is None: lang = entry['meta']['languages'].lower() if lang == 'r': utils.install_r_deps(deplist, model, source='cran') elif 'python'.startswith(lang): utils.install_python_deps(deplist, model, source='pip') # ----- System deps ----- elif category == 'system' or 'shell'.startswith(category): utils.install_system_deps(deplist) # ----- R deps ----- elif category == 'r': utils.install_r_deps(deplist, model, source='cran') elif category == 'cran' or category == 'github' or category.startswith( 'cran-'): utils.install_r_deps(deplist, model, source=category) # ----- Python deps ----- elif category.startswith('python') or category.startswith( 'pip') or category == 'conda': utils.install_python_deps(deplist, model, source=category) # ----- Files ----- elif 'files'.startswith(category): utils.install_file_deps(deplist, model) # Run additional configure script if any. conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf if not conf: if depspec is not None: msg = ("No configuration script provided for this model. " "The following dependencies are required:\n") print(msg) print(yaml.dump(depspec)) else: print("No configuration provided (maybe none is required).") # Update working dir if any. if args.workding_dir is not None: utils.update_working_dir(model, args.workding_dir) # Suggest next step. if not args.quiet: utils.print_next_step('configure', model=model)
def configure_model(args): """Ensure the user's environment is configured.""" # TODO: Add support for additional configuration if any except those # specified in MLHUB.yaml. # TODO: When fail, print out the failed dep, as well as installed # deps and non-installed deps. # TODO: Add support for specifying packages version. # TODO: Add more informative messages for different kinds of # dependencies. # Other ideas for configuration # # 1 Construct mlhub container (from Ubuntu) with known starting point # # 2 Assume the user is on a DSVM with free Azure account to test out. # # 3 Read dependencies: and language: and then install as required: # # 4 Assume model packager provides a configure.R script. This is a # override and no other configuration happens if this is # supplied. Alternatively this is viewed as a cop-out providing # no support from mlhub for the model packager. The preference # would be to use the dependencies: tag to list the required R # or python packages. # # So the meta-code might be # # if file.exists(configure.XX): # XX configure.XX # else if language: == "Rscript": # packages <- dependencies: # install <- packages[!(packages %in% installed.packages()[,"Package"])] # if(length(new.packages)) install.packages(install) # else if language: == "python": # packages = dependencies: # cat packages > requirements.txt # pip install -r requirements.txt # YES = args.y | args.yes # Avoid 403 errors which result when the header identifies itself # as python urllib or is empty and thus the web site assumes it is # a robot. We are not a robot but a user downloading a file. This # will ensure gitlab is okay with retrieving from a URL by adding # a header rather than no header. TODO move to using Requests. opener = urllib.request.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] urllib.request.install_opener(opener) if not args.model: # Configure MLHUB per se. # Includes bash completion and system pre-requisites if distro.id() in ["debian", "ubuntu"]: path = os.path.dirname(__file__) env_var = "export _MLHUB_OPTION_YES='y'; " if YES else "" env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable) script = os.path.join("scripts", "dep", "mlhub.sh") command = "{}{} {}".format(env_var, BASH_CMD, script) proc = subprocess.Popen( command, shell=True, cwd=path, stderr=subprocess.PIPE ) output, errors = proc.communicate() if proc.returncode != 0: raise utils.ConfigureFailedException(errors.decode("utf-8")) return # Model package configuration. model = args.model # Correct model name if possible. matched_model = utils.get_misspelled_pkg(model) if matched_model is not None: model = matched_model # Setup. pkg_dir = utils.get_package_dir(model) # Check if the model package is installed. utils.check_model_installed(model) # Install dependencies specified in MLHUB.yaml entry = utils.load_description(model) depspec = None if "dependencies" in entry: depspec = entry["dependencies"] elif "dependencies" in entry["meta"]: depspec = entry["meta"]["dependencies"] if depspec is not None: for spec in utils.flatten_mlhubyaml_deps(depspec): category = spec[0][-1] deplist = spec[1] # Category include: # ------------------------------------------------------------------------------ # category | action # -----------------|------------------------------------------------------------ # None | install package according to entry['meta']['languages'] # | if R, install.packages(xxx) from cran; # | if Python, pip install xxx # -----------------|------------------------------------------------------------ # system | apt-get install # sh | apt-get install # -----------------|------------------------------------------------------------ # r | install.packages(xxx) from cran, version can be specified # cran | install.packages(xxx) from cran, version can be specified # cran-2018-12-01 | install cran snapshot on 2018-12-01 # github | devtools::install_github from github # -----------------|------------------------------------------------------------ # python | apt-get install python-xxx # python3 | apt-get install python3-xxx # pip | pip install # pip3 | pip3 install # conda | conda install # -----------------|------------------------------------------------------------ # files | download files # -----------------|------------------------------------------------------------ # ----- Determine deps by language ----- if category is None: lang = entry["meta"]["languages"].lower() if lang == "r": utils.install_r_deps( deplist, model, source="cran", yes=YES ) elif "python".startswith(lang): utils.install_python_deps( deplist, model, source="pip", yes=YES ) # ----- System deps ----- elif category == "system" or "shell".startswith(category): utils.install_system_deps(deplist, yes=YES) # ----- R deps ----- elif category == "r": utils.install_r_deps(deplist, model, source="cran", yes=YES) elif ( category == "cran" or category == "github" or category.startswith("cran-") ): utils.install_r_deps(deplist, model, source=category, yes=YES) # ----- Python deps ----- elif ( category.startswith("python") or category.startswith("pip") or category == "conda" ): utils.install_python_deps( deplist, model, source=category, yes=YES ) # ----- Files ----- elif "files".startswith(category): utils.install_file_deps(deplist, model, key=args.i, yes=YES) # Run additional configure script if any. conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf if not conf: if depspec is not None: msg = ( "No configuration script provided for this model. " "The following dependencies are required:\n" ) print(msg) print(yaml.dump(depspec)) else: print("No configuration provided (maybe none is required).") # Update working dir if any. if args.working_dir is not None: utils.update_working_dir(model, args.working_dir) # Suggest next step. if not args.quiet: utils.print_next_step("configure", model=model)