def readme(args): """Display the model's README information.""" model = args.model # Correct model name if possible. matched_model = utils.get_misspelled_pkg(model) if matched_model is not None: model = matched_model # Setup. logger = logging.getLogger(__name__) logger.info("Get README of {}.".format(model)) path = utils.get_package_dir(model) readme_file = os.path.join(path, README) # Check that the model is installed. utils.check_model_installed(model) # Display the README. if not os.path.exists( readme_file): # Try to generate README from README.md readme_raw = readme_file[:readme_file.rfind('.')] + '.md' if not os.path.exists(readme_raw): readme_raw = readme_raw[:readme_raw.rfind('.')] + '.rst' if not os.path.exists(readme_raw): raise utils.ModelReadmeNotFoundException(model, readme_file) script = os.path.join(os.path.dirname(__file__), 'scripts', 'convert_readme.sh') command = "/bin/bash {} {} {}".format(script, readme_raw, README) proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() if proc.returncode != 0: errors = errors.decode("utf-8") command_not_found = re.compile(r"\d: (.*):.*not found").search( errors) if command_not_found is not None: raise utils.LackPrerequisiteException( command_not_found.group(1)) print("An error was encountered:\n") print(errors) raise utils.ModelReadmeNotFoundException(model, readme_file) with open(readme_file, 'r') as f: print(utils.drop_newline(f.read())) # Suggest next step. if not args.quiet: utils.print_next_step('readme', model=model)
def remove_model(args): """Remove installed model.""" # Setup. model = args.model if model is None: if os.path.exists(MLINIT): path = MLINIT msg = "*Completely* remove all installed models in '{}' [y/N]? " else: msg = "The local model folder '{}' does not exist. Nothing to do." msg = msg.format(MLINIT) print(msg) sys.exit(1) else: path = MLINIT + model msg = "Remove '{}' [y/N]? " # Check that the model is installed. utils.check_model_installed(model) sys.stdout.write(msg.format(path)) choice = input().lower() if choice == 'y': rmtree(path) else: if model is None and not args.quiet: utils.print_next_step('remove')
def remove_model(args): """Remove installed model.""" # TODO: Remove .archive and .config for the model. model = args.model # Determine if remove all model pkgs or a certain model pkg. cache = None if model is None: path = utils.get_init_dir() if os.path.exists(path): msg = "*Completely* remove all installed models in '{}'" else: msg = "The local model folder '{}' does not exist. Nothing to do." msg = msg.format(path) print(msg) return else: # Correct model name if possible. matched_model = utils.get_misspelled_pkg(model) if matched_model is not None: model = matched_model path = utils.get_package_dir(model) if os.path.exists(utils.get_package_cache_dir(model)): cache = utils.get_package_cache_dir(model) msg = "Remove '{}/'" # Check that the model is installed. utils.check_model_installed(model) if utils.yes_or_no(msg, path, yes=True): # Remove package installation dir shutil.rmtree(path) # Remove package config dir as well without ask path = utils.get_package_config_dir(model) if os.path.exists(path): shutil.rmtree(path) # Ask if remove cached files if cache is not None and utils.yes_or_no( "Remove cache '{}/' as well", cache, yes=False ): shutil.rmtree(cache) archive = utils.get_package_archive_dir(model) if os.path.exists(archive): shutil.rmtree(archive) else: if model is None and not args.quiet: utils.print_next_step("remove")
def list_model_commands(args): """ List the commands supported by this model.""" model = args.model # Correct model name if possible. matched_model = utils.get_misspelled_pkg(model) if matched_model is not None: model = matched_model logger = logging.getLogger(__name__) logger.info("List available commands of '{}'".format(model)) # Check that the model is installed. utils.check_model_installed(model) entry = utils.load_description(model) commands = entry['commands'] if args.name_only: print('\n'.join(list(commands))) return msg = "The '{}' model " meta = entry['meta'] if 'title' not in meta: title = None else: title = utils.lower_first_letter(utils.dropdot(meta['title'])) msg += "({}) " msg += "supports the following commands:" msg = msg.format(model, title) msg = textwrap.fill(msg, width=75) print(msg) for cmd in commands: utils.print_model_cmd_help(entry, cmd) # Update bash completion list. utils.update_command_completion(set(commands)) # Suggest next step. if not args.quiet: utils.print_next_step('commands', description=entry, model=model)
def download_model(args): """Download the large pre-built model.""" # TODO: Will this be a url in the DESCRIPTION file or will it be a # download.sh script. Which ever (maybe either), if it is present # then this command is available and will download the required # file, perhaps from the actual source of the model. model = args.model # Check that the model is installed. utils.check_model_installed(model) if not args.quiet: utils.print_next_step('download', model=model)
def dispatch(args): """Dispatch other commands to the appropriate model provided script.""" cmd = args.cmd model = args.model path = MLINIT + model param = " ".join(args.param) # Check that the model is installed. utils.check_model_installed(model) desc = utils.load_description(model) # Obtain the specified script file. script = desc["commands"][cmd]["script"].split(" ")[0] + " " + param # Determine the interpreter to use # # .R => Rscript; .py => python, etc. (root, ext) = os.path.splitext(script) ext = ext.strip() if ext == ".R": interpreter = "Rscript" elif ext == ".py": interpreter = "python" else: msg = "Could not determine an interpreter for extension '{}'".format( ext) print(msg, file=sys.stderr) sys.exit() command = "{} {}".format(interpreter, script) if args.debug: print(DEBUG + "(cd " + path + "; " + command + ")") proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() if proc.returncode != 0: print("An error was encountered:\n") print(errors.decode("utf-8"))
def download_model(args): """Download the large pre-built model.""" # TODO: Will this be a url in the DESCRIPTION file or will it be a # download.sh script. Which ever (maybe either), if it is present # then this command is available and will download the required # file, perhaps from the actual source of the model. model = args.model # Check that the model is installed. utils.check_model_installed(model) if not args.quiet: msg = "Model information is available:\n\n $ {} readme {}\n" msg = msg.format(CMD, model) print(msg)
def readme(args): """Display the model's README information.""" # Setup. model = args.model path = MLINIT + model readme = os.path.join(path, README) # Check that the model is installed. utils.check_model_installed(model) # Display the README. with open(readme, 'r') as f: print(utils.drop_newline(f.read())) # Suggest next step. if not args.quiet: utils.print_next_step('readme', model=model)
def rename_model(args): """Rename an installed model. OLD is the old package name and NEW the new one. If the NEW already exists it is overwritten only if --force. """ old = args.old new = args.new utils.check_model_installed(old) oldp = utils.get_package_dir(old) newp = utils.get_package_dir(new) if os.path.exists(newp): if args.force: shutil.rmtree(newp) else: raise utils.ModelInstalledException(new) os.rename(oldp, newp) yfile = f"{newp}/{constants.MLHUB_YAML}" with open(yfile) as file: ydata = yaml.load(file, Loader=yaml.FullLoader) ydata["meta"]["name"] = new with open(yfile, "w") as file: yaml.dump(ydata, file, sort_keys=False) if not args.quiet: msg = f"Renamed '{old}' as '{new}' (now '{newp}')." print(msg)
def list_model_commands(args): """ List the commands supported by this model.""" model = args.model # Check that the model is installed. utils.check_model_installed(model) info = utils.load_description(model) msg = "The model '{}' ({}) supports the following commands:" msg = msg.format(model, info['meta']['title']) msg = textwrap.fill(msg, width=60) print(msg + "\n") yaml.dump(info['commands'], sys.stdout, default_flow_style=False) # Suggest next step. if not args.quiet: msg = "Model dependencies are listed using:\n\n $ {} configure {}\n" msg = msg.format(CMD, model) print(msg)
def readme(args): """Display the model's README information.""" # Setup. model = args.model path = MLINIT + model readme = os.path.join(path, README) # Check that the model is installed. utils.check_model_installed(model) # Display the README. with open(readme, 'r') as f: print(f.read()) # Suggest next step. if not args.quiet: msg = "Model dependencies are listed using:\n\n $ {} configure {}\n" msg = msg.format(CMD, model) print(msg)
def dispatch(args): """Dispatch other commands to the appropriate model provided script.""" cmd = args.cmd model = args.model path = utils.get_package_dir(model) param = " ".join(args.param) # Get working dir if any. if args.workding_dir is not None: utils.update_working_dir(model, args.workding_dir) if args.workding_dir == '': args.workding_dir = None else: args.working_dir = utils.get_working_dir(model) # Get conda environment name if any. conda_env_name = utils.get_conda_env_name(model) # Check that the model is installed and has commands. utils.check_model_installed(model) entry = utils.load_description(model) if 'commands' not in entry or len(entry['commands']) == 0: raise utils.CommandNotFoundException(cmd, model) # Correct misspelled command if possible. matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands'])) if matched_cmd is not None: cmd = matched_cmd # Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml. meta = entry['meta'] if 'display' in meta and cmd in meta['display'] and os.environ.get( 'DISPLAY', '') == '': msg = "Graphic display is required but not available for command '{}'. Continue" yes = utils.yes_or_no(msg, cmd, yes=False) if not yes: msg = """ To enable DISPLAY be sure to connect to the server using 'ssh -X' or else connect to the server's desktop using a local X server like X2Go. """ sys.stdout.write(msg) sys.exit(1) # Obtain the default/chosen language for the package. lang = meta["languages"] # Deal with malformed 'languages' field lang_opts = {"python": "py", "R": "R"} for k in list(lang_opts): if lang in k: lang = lang_opts[k] break # Obtain the specified script file. script = cmd + "." + lang logger = logging.getLogger(__name__) logger.debug("Execute the script: " + os.path.join(path, script)) if cmd not in list(entry['commands']) or not os.path.exists( os.path.join(path, script)): raise utils.CommandNotFoundException(cmd, model) # Determine the interpreter to use # # .R => Rscript; .py => python, etc. interpreter = utils.interpreter(script) # Change working dir if needed if args.workding_dir is not None: script = os.path.join(path, script) path = args.workding_dir # _MLHUB_CMD_CWD: a environment variable indicates current working # directory where command `ml xxx` is invoked. # _MLHUB_MODEL_NAME: env variable indicates the name of the model. # # The above two env vars can be obtained by helper function, such # as utils.get_cmd_cwd(). And model package developer should be # use the helper function instead of the env vars directly. command = "export _MLHUB_CMD_CWD='{}'; export _MLHUB_MODEL_NAME='{}'; {} {} {}".format( os.getcwd(), model, interpreter, script, param) # Run script inside conda environment if specified if conda_env_name is not None: command = 'bash -c "source activate {}; {}"'.format( conda_env_name, command) logger.debug("(cd " + path + "; " + command + ")") proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() missing_r_dep = False if proc.returncode != 0: errors = errors.decode("utf-8") # Check if it is Python dependency unsatisfied dep_required = re.compile( r"ModuleNotFoundError: No module named '(.*)'").search(errors) # Check if R dependency unsatisified if dep_required is None: dep_required = re.compile( r"there is no package called ‘(.*)’").search(errors) if dep_required is not None: missing_r_dep = True # Check if required data resource not found data_required = re.compile( r"mlhub.utils.DataResourceNotFoundException").search(errors) if dep_required is not None: # Dependency unsatisfied dep_required = dep_required.group(1) logger.error("Dependency unsatisfied: {}\n{}".format( dep_required, errors)) raise utils.LackDependencyException(dep_required, missing_r_dep, model) elif data_required is not None: # Data not found raise utils.DataResourceNotFoundException() else: # Other unknown errors print("An error was encountered:\n") print(errors) else: # Suggest next step if not args.quiet: utils.print_next_step(cmd, description=entry, model=model)
def configure_model(args): """Ensure the user's environment is configured.""" # TODO: Add support for additional configuration if any except those # specified in MLHUB.yaml. # TODO: When fail, print out the failed dep, as well as installed # deps and non-installed deps. # TODO: Add support for specifying packages version. # TODO: Add more informative messages for different kinds of # dependencies. # Other ideas for configuration # # 1 Construct mlhub container (from Ubuntu) with known starting point # # 2 Assume the user is on a DSVM with free Azure account to test out. # # 3 Read dependencies: and language: and then install as required: # # 4 Assume model packager provides a configure.R script. This is a # override and no other configuration happens if this is # supplied. Alternatively this is viewed as a cop-out prividing # no support from mlhub for the model packager. The preference # would be to use the dependencies: tag to list the required R # or python packages. # # So the meta-code might be # # if file.exists(configure.XX): # XX configure.XX # else if language: == "Rscript": # packages <- dependencies: # install <- packages[!(packages %in% installed.packages()[,"Package"])] # if(length(new.packages)) install.packages(install) # else if language: == "python": # packages = dependencies: # cat pacakges > requirements.txt # pip install -r requirements.txt # if not args.model: # Configure MLHUB per se. # Includes bash completion and system pre-requisites if distro.id() in ['debian', 'ubuntu']: path = os.path.dirname(__file__) command = '/bin/bash {}'.format( os.path.join('scripts', 'dep', 'mlhub.sh')) proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() if proc.returncode != 0: errors = errors.decode("utf-8") print("\nAn error was encountered:\n") print(errors) raise utils.ConfigureFailedException() return # Model package configuration. model = args.model # Correct model name if possible. matched_model = utils.get_misspelled_pkg(model) if matched_model is not None: model = matched_model # Setup. pkg_dir = utils.get_package_dir(model) # Check if the model package is installed. utils.check_model_installed(model) # Install dependencies specified in MLHUB.yaml entry = utils.load_description(model) depspec = None if 'dependencies' in entry: depspec = entry['dependencies'] elif 'dependencies' in entry['meta']: depspec = entry['meta']['dependencies'] if depspec is not None: for spec in utils.flatten_mlhubyaml_deps(depspec): category = spec[0][-1] deplist = spec[1] # Category include: # ------------------------------------------------------------------------------ # category | action # -----------------|------------------------------------------------------------ # None | install package according to entry['meta']['languages'] # | if R, install.packages(xxx) from cran; # | if Python, pip install xxx # -----------------|------------------------------------------------------------ # system | apt-get install # sh | apt-get install # -----------------|------------------------------------------------------------ # r | install.packages(xxx) from cran, version can be specified # cran | install.packages(xxx) from cran, version can be specified # cran-2018-12-01 | install cran snapshot on 2018-12-01 # github | devtools::install_github from github # -----------------|------------------------------------------------------------ # python | apt-get install python-xxx # python3 | apt-get install python3-xxx # pip | pip install # pip3 | pip3 install # conda | conda install # -----------------|------------------------------------------------------------ # files | download files # -----------------|------------------------------------------------------------ # ----- Determine deps by language ----- if category is None: lang = entry['meta']['languages'].lower() if lang == 'r': utils.install_r_deps(deplist, model, source='cran') elif 'python'.startswith(lang): utils.install_python_deps(deplist, model, source='pip') # ----- System deps ----- elif category == 'system' or 'shell'.startswith(category): utils.install_system_deps(deplist) # ----- R deps ----- elif category == 'r': utils.install_r_deps(deplist, model, source='cran') elif category == 'cran' or category == 'github' or category.startswith( 'cran-'): utils.install_r_deps(deplist, model, source=category) # ----- Python deps ----- elif category.startswith('python') or category.startswith( 'pip') or category == 'conda': utils.install_python_deps(deplist, model, source=category) # ----- Files ----- elif 'files'.startswith(category): utils.install_file_deps(deplist, model) # Run additional configure script if any. conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf if not conf: if depspec is not None: msg = ("No configuration script provided for this model. " "The following dependencies are required:\n") print(msg) print(yaml.dump(depspec)) else: print("No configuration provided (maybe none is required).") # Update working dir if any. if args.workding_dir is not None: utils.update_working_dir(model, args.workding_dir) # Suggest next step. if not args.quiet: utils.print_next_step('configure', model=model)
def configure_model(args): """Ensure the user's environment is configured.""" # TODO: Install packages natively for those listed in # dependencies. Then if there is also a configure.sh, then run # that for additoinal setup. # Other ideas re cofiguration # # 1 Construct mlhub container (from Ubuntu) with known starting point # # 2 Assume the user is on a DSVM with free Azure account to test out. # # 3 Read dependencies: and language: and then install as required: # # 4 Assume model packager provides a configure.R script. This is a # override and no other configuration happens if this is # supplied. Alternatively this is viewed as a cop-out prividing # no support from mlhub for the model packager. The preference # would be to use the dependencies: tag to list the required R # or python packages. # # So the meta-code might be # # if file.exists(configure.XX): # XX configure.XX # else if language: == "Rscript": # packages <- dependencies: # install <- packages[!(packages %in% installed.packages()[,"Package"])] # if(length(new.packages)) install.packages(install) # else if language: == "python": # packages = dependencies: # cat pacakges > requirements.txt # pip install -r requirements.txt # # Setup. model = args.model path = MLINIT + model # Check that the model is installed. utils.check_model_installed(model) # If there is a configure script and this is a Ubuntu host, then # run the configure script. conf = os.path.join(path, "configure.sh") if platform.dist()[0] in set(['debian', 'Ubuntu' ]) and os.path.exists(conf): command = "bash configure.sh" if not args.quiet: msg = "Configuration will take place using '{}'.\n" msg = msg.format(conf) print(msg) proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() if proc.returncode != 0: print("An error was encountered:\n") print(errors.decode("utf-8")) else: # For now simply list the declared dependencies for the user to # make sure they have it all installed. if not args.quiet: msg = """ Configuration is yet to be automated. The following dependencies are required: """ print(msg) info = utils.load_description(model) msg = info["meta"]["dependencies"] print(" ====> \033[31m" + msg + "\033[0m") # Suggest next step. if not args.quiet: msg = "\nOnce configured run the demonstration:\n\n $ {} demo {}\n" msg = msg.format(CMD, model) print(msg)
def dispatch(args): """Dispatch other commands to the appropriate model provided script.""" cmd = args.cmd model = args.model path = utils.get_package_dir(model) param = " ".join(args.param) # Get working dir if any. if args.workding_dir is not None: utils.update_working_dir(model, args.workding_dir) if args.workding_dir == '': args.workding_dir = None else: args.working_dir = utils.get_working_dir(model) # Get conda environment name if any. conda_env_name = utils.get_conda_env_name(model) # Check that the model is installed and has commands. utils.check_model_installed(model) entry = utils.load_description(model) if 'commands' not in entry or len(entry['commands']) == 0: raise utils.CommandNotFoundException(cmd, model) # Correct misspelled command if possible. matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands'])) if matched_cmd is not None: cmd = matched_cmd # Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml. meta = entry['meta'] if 'display' in meta and cmd in meta['display'] and os.environ.get( 'DISPLAY', '') == '': msg = "Graphic display is required but not available for command '{}'. Continue" yes = utils.yes_or_no(msg, cmd, yes=False) if not yes: msg = """ To enable DISPLAY be sure to connect to the server using 'ssh -X' or else connect to the server's desktop using a local X server like X2Go. """ sys.stdout.write(msg) sys.exit(1) # Obtain the default/chosen language for the package. lang = meta["languages"] # Deal with malformed 'languages' field lang_opts = {"python": "py", "R": "R"} for k in list(lang_opts): if lang in k: lang = lang_opts[k] break # Obtain the specified script file. script = cmd + "." + lang logger = logging.getLogger(__name__) logger.debug("Execute the script: " + os.path.join(path, script)) if cmd not in list(entry['commands']) or not os.path.exists( os.path.join(path, script)): raise utils.CommandNotFoundException(cmd, model) # Determine the interpreter to use # # .R => Rscript; .py => python, etc. interpreter = utils.interpreter(script) # Change working dir if needed if args.workding_dir is not None: script = os.path.join(path, script) path = args.workding_dir # Handle python environment python_pkg_bin = None python_pkg_path = None if script.endswith('py'): python_pkg_base = os.path.sep.join( [utils.get_package_dir(model), '.python']) python_pkg_path = python_pkg_base + site.USER_SITE python_pkg_bin = python_pkg_base + site.USER_BASE + '/bin' # TODO: Make sure to document: # $ sudo apt-get install -y python3-pip # $ /usr/bin/pip3 install mlhub # Since in DSVM, the default pip is conda's pip, so if we stick to # use system's command, then the installation of MLHub itself should # be completed via system's pip, otherwise, MLHub will not work. if sys.executable != SYS_PYTHON_CMD: python_pkg_path = python_pkg_base + site.getsitepackages()[0] python_pkg_bin = python_pkg_base + site.PREFIXES[0] + '/bin' if utils.get_sys_python_pkg_usage(model): utils.print_on_stderr(MSG_INCOMPATIBLE_PYTHON_ENV, model) # _MLHUB_CMD_CWD: a environment variable indicates current working # directory where command `ml xxx` is invoked. # _MLHUB_MODEL_NAME: env variable indicates the name of the model. # # The above two env vars can be obtained by helper function, such # as utils.get_cmd_cwd(). And model package developer should be # use the helper function instead of the env vars directly. env_var = "export _MLHUB_CMD_CWD='{}'; ".format(os.getcwd()) env_var += "export _MLHUB_MODEL_NAME='{}'; ".format(model) env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable) env_var += "export PYTHONPATH='{}'; ".format( python_pkg_path) if python_pkg_path else "" env_var += "export PATH=\"{}:$PATH\"; ".format( python_pkg_bin) if python_pkg_bin else "" command = "{}{} {} {}".format(env_var, interpreter, script, param) # Run script inside conda environment if specified if conda_env_name is not None: command = '{} -c "source activate {}; {}"'.format( BASH_CMD, conda_env_name, command) logger.debug("(cd " + path + "; " + command + ")") proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() missing_r_dep = False if proc.returncode != 0: errors = errors.decode("utf-8") # Check if it is Python dependency unsatisfied dep_required = re.compile( r"ModuleNotFoundError: No module named '(.*)'").search(errors) # Check if R dependency unsatisified if dep_required is None: dep_required = re.compile( r"there is no package called ‘(.*)’").search(errors) if dep_required is not None: missing_r_dep = True # Check if required data resource not found data_required = re.compile( r"mlhub.utils.DataResourceNotFoundException").search(errors) if dep_required is not None: # Dependency unsatisfied dep_required = dep_required.group(1) logger.error("Dependency unsatisfied: {}\n{}".format( dep_required, errors)) raise utils.LackDependencyException(dep_required, missing_r_dep, model) elif data_required is not None: # Data not found raise utils.DataResourceNotFoundException() else: # Other unknown errors print("An error was encountered:\n") print(errors) else: # Suggest next step if not args.quiet: utils.print_next_step(cmd, description=entry, model=model)
def dispatch(args): """Dispatch other commands to the appropriate model provided script.""" cmd = args.cmd model = args.model path = MLINIT + model param = " ".join(args.param) # Check that the model is installed. utils.check_model_installed(model) desc = utils.load_description(model) # Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml. if 'display' in desc['meta'] and cmd in desc['meta']['display'] and os.environ.get('DISPLAY', '') == '': msg = "Graphic display is required but not available for command '{}'. Continue [y/N]? " msg = msg.format(cmd) sys.stdout.write(msg) choice = input().lower() if choice != 'y': msg = """ To enable DISPLAY be sure to connect to the server using 'ssh -X' or else connect to the server's desktop using a local X server like X2Go. """ sys.stdout.write(msg) sys.exit(1) # Obtain the default/chosen language for the package. lang = desc["meta"]["languages"] # Deal with malformed 'languages' field lang_opts = {"python": "py", "R": "R"} for k in list(lang_opts): if lang in k: lang = lang_opts[k] break # Obtain the specified script file. script = cmd + "." + lang if args.debug: print(DEBUG + "execute the script: " + os.path.join(path, script)) if cmd not in list(desc['commands']) or not os.path.exists(os.path.join(path, script)): msg = """{}The command '{}' was not found for this model. Try using 'commands' to list all supported commands: $ {} commands {} """.format(APPX, cmd, CMD, model) print(msg, file=sys.stderr) sys.exit(1) # Determine the interpreter to use # # .R => Rscript; .py => python, etc. interpreter = utils.interpreter(script) command = "export CMD_CWD='{}'; {} {} {}".format(os.getcwd(), interpreter, script, param) if args.debug: print(DEBUG + "(cd " + path + "; " + command + ")") proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() if proc.returncode != 0: print("An error was encountered:\n") print(errors.decode("utf-8")) else: # Suggest next step if not args.quiet: utils.print_next_step(cmd, description=desc, model=model)
def configure_model(args): """Ensure the user's environment is configured.""" # TODO: Install packages natively for those listed in # dependencies. Then if there is also a configure.sh, then run # that for additoinal setup. # Other ideas re cofiguration # # 1 Construct mlhub container (from Ubuntu) with known starting point # # 2 Assume the user is on a DSVM with free Azure account to test out. # # 3 Read dependencies: and language: and then install as required: # # 4 Assume model packager provides a configure.R script. This is a # override and no other configuration happens if this is # supplied. Alternatively this is viewed as a cop-out prividing # no support from mlhub for the model packager. The preference # would be to use the dependencies: tag to list the required R # or python packages. # # So the meta-code might be # # if file.exists(configure.XX): # XX configure.XX # else if language: == "Rscript": # packages <- dependencies: # install <- packages[!(packages %in% installed.packages()[,"Package"])] # if(length(new.packages)) install.packages(install) # else if language: == "python": # packages = dependencies: # cat pacakges > requirements.txt # pip install -r requirements.txt # if not args.model: # Configure ml. Currently only bash completion. import platform sys_version = platform.uname().version.lower() if 'debian' in sys_version or 'ubuntu' in sys_version: path = os.path.dirname(__file__) commands = [ 'sudo cp {} /etc/bash_completion.d'.format(COMPLETION_SCRIPT), 'ml available > /dev/null', 'ml installed > /dev/null', ] for cmd in commands: print('Executing: ', cmd) subprocess.run(cmd, shell=True, cwd=path, stderr=subprocess.PIPE) print("\nLast step to make tab completion take effect: \n\n $ source /etc/bash_completion.d/ml.bash") return # Setup. model = args.model path = MLINIT + model # Check that the model is installed. utils.check_model_installed(model) # If there are any configure scripts then run them, else print the # list of supplied dependencies if any. Note that Python's 'or' is # lazy evaluation. conf = utils.configure(path, "configure.sh", args.quiet) conf = utils.configure(path, "configure.R", args.quiet) or conf conf = utils.configure(path, "configure.py", args.quiet) or conf if not conf: try: info = utils.load_description(model) deps = info["meta"]["dependencies"] if not args.quiet: msg = "No configuration script provided for this model. " msg = msg + "The following dependencies are required:\n" print(msg) print(" ====> \033[31m" + deps + "\033[0m") except: print("No configuration provided (maybe none is required).") # Suggest next step. if not args.quiet: utils.print_next_step('configure', model=model)
def list_model_commands(args): """ List the commands supported by this model.""" # Setup. model = args.model # Check that the model is installed. utils.check_model_installed(model) info = utils.load_description(model) if args.name_only: print('\n'.join(list(info['commands']))) return msg = "The model '{}' " if 'title' not in info['meta']: title = None else: title = utils.lower_first_letter(utils.dropdot(info['meta']['title'])) msg += "({}) " msg += "supports the following commands:" msg = msg.format(model, title) msg = textwrap.fill(msg, width=75) print(msg) for c in info['commands']: print("\n $ {} {} {}".format(CMD, c, model)) c_meta = info['commands'][c] if type(c_meta) is str: print(" " + c_meta) else: # Handle malformed DESCRIPTION.yaml like # -- # commands: # print: # description: print a textual summary of the model # score: # equired: the name of a CSV file containing a header and 6 columns # description: apply the model to a supplied dataset desc = c_meta.get('description', None) if desc is not None: print(" " + desc) c_meta = {k:c_meta[k] for k in c_meta if k != 'description'} if len(c_meta) > 0: msg = yaml.dump(c_meta, default_flow_style=False) msg = msg.split('\n') msg = [" " + ele for ele in msg] print('\n'.join(msg), end='') # Update available commands for the model for fast bash tab completion. utils.update_completion_list(COMPLETION_COMMANDS, set(info['commands'])) # Suggest next step. if not args.quiet: utils.print_next_step('commands', description=info, model=model)
def configure_model(args): """Ensure the user's environment is configured.""" # TODO: Add support for additional configuration if any except those # specified in MLHUB.yaml. # TODO: When fail, print out the failed dep, as well as installed # deps and non-installed deps. # TODO: Add support for specifying packages version. # TODO: Add more informative messages for different kinds of # dependencies. # Other ideas for configuration # # 1 Construct mlhub container (from Ubuntu) with known starting point # # 2 Assume the user is on a DSVM with free Azure account to test out. # # 3 Read dependencies: and language: and then install as required: # # 4 Assume model packager provides a configure.R script. This is a # override and no other configuration happens if this is # supplied. Alternatively this is viewed as a cop-out providing # no support from mlhub for the model packager. The preference # would be to use the dependencies: tag to list the required R # or python packages. # # So the meta-code might be # # if file.exists(configure.XX): # XX configure.XX # else if language: == "Rscript": # packages <- dependencies: # install <- packages[!(packages %in% installed.packages()[,"Package"])] # if(length(new.packages)) install.packages(install) # else if language: == "python": # packages = dependencies: # cat packages > requirements.txt # pip install -r requirements.txt # YES = args.y | args.yes # Avoid 403 errors which result when the header identifies itself # as python urllib or is empty and thus the web site assumes it is # a robot. We are not a robot but a user downloading a file. This # will ensure gitlab is okay with retrieving from a URL by adding # a header rather than no header. TODO move to using Requests. opener = urllib.request.build_opener() opener.addheaders = [('User-agent', 'Mozilla/5.0')] urllib.request.install_opener(opener) if not args.model: # Configure MLHUB per se. # Includes bash completion and system pre-requisites if distro.id() in ["debian", "ubuntu"]: path = os.path.dirname(__file__) env_var = "export _MLHUB_OPTION_YES='y'; " if YES else "" env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable) script = os.path.join("scripts", "dep", "mlhub.sh") command = "{}{} {}".format(env_var, BASH_CMD, script) proc = subprocess.Popen( command, shell=True, cwd=path, stderr=subprocess.PIPE ) output, errors = proc.communicate() if proc.returncode != 0: raise utils.ConfigureFailedException(errors.decode("utf-8")) return # Model package configuration. model = args.model # Correct model name if possible. matched_model = utils.get_misspelled_pkg(model) if matched_model is not None: model = matched_model # Setup. pkg_dir = utils.get_package_dir(model) # Check if the model package is installed. utils.check_model_installed(model) # Install dependencies specified in MLHUB.yaml entry = utils.load_description(model) depspec = None if "dependencies" in entry: depspec = entry["dependencies"] elif "dependencies" in entry["meta"]: depspec = entry["meta"]["dependencies"] if depspec is not None: for spec in utils.flatten_mlhubyaml_deps(depspec): category = spec[0][-1] deplist = spec[1] # Category include: # ------------------------------------------------------------------------------ # category | action # -----------------|------------------------------------------------------------ # None | install package according to entry['meta']['languages'] # | if R, install.packages(xxx) from cran; # | if Python, pip install xxx # -----------------|------------------------------------------------------------ # system | apt-get install # sh | apt-get install # -----------------|------------------------------------------------------------ # r | install.packages(xxx) from cran, version can be specified # cran | install.packages(xxx) from cran, version can be specified # cran-2018-12-01 | install cran snapshot on 2018-12-01 # github | devtools::install_github from github # -----------------|------------------------------------------------------------ # python | apt-get install python-xxx # python3 | apt-get install python3-xxx # pip | pip install # pip3 | pip3 install # conda | conda install # -----------------|------------------------------------------------------------ # files | download files # -----------------|------------------------------------------------------------ # ----- Determine deps by language ----- if category is None: lang = entry["meta"]["languages"].lower() if lang == "r": utils.install_r_deps( deplist, model, source="cran", yes=YES ) elif "python".startswith(lang): utils.install_python_deps( deplist, model, source="pip", yes=YES ) # ----- System deps ----- elif category == "system" or "shell".startswith(category): utils.install_system_deps(deplist, yes=YES) # ----- R deps ----- elif category == "r": utils.install_r_deps(deplist, model, source="cran", yes=YES) elif ( category == "cran" or category == "github" or category.startswith("cran-") ): utils.install_r_deps(deplist, model, source=category, yes=YES) # ----- Python deps ----- elif ( category.startswith("python") or category.startswith("pip") or category == "conda" ): utils.install_python_deps( deplist, model, source=category, yes=YES ) # ----- Files ----- elif "files".startswith(category): utils.install_file_deps(deplist, model, key=args.i, yes=YES) # Run additional configure script if any. conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf if not conf: if depspec is not None: msg = ( "No configuration script provided for this model. " "The following dependencies are required:\n" ) print(msg) print(yaml.dump(depspec)) else: print("No configuration provided (maybe none is required).") # Update working dir if any. if args.working_dir is not None: utils.update_working_dir(model, args.working_dir) # Suggest next step. if not args.quiet: utils.print_next_step("configure", model=model)