def dispatch(args): """Dispatch other commands to the appropriate model provided script.""" cmd = args.cmd model = args.model path = MLINIT + model param = " ".join(args.param) # Check that the model is installed. utils.check_model_installed(model) desc = utils.load_description(model) # Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml. if 'display' in desc['meta'] and cmd in desc['meta']['display'] and os.environ.get('DISPLAY', '') == '': msg = "Graphic display is required but not available for command '{}'. Continue [y/N]? " msg = msg.format(cmd) sys.stdout.write(msg) choice = input().lower() if choice != 'y': msg = """ To enable DISPLAY be sure to connect to the server using 'ssh -X' or else connect to the server's desktop using a local X server like X2Go. """ sys.stdout.write(msg) sys.exit(1) # Obtain the default/chosen language for the package. lang = desc["meta"]["languages"] # Deal with malformed 'languages' field lang_opts = {"python": "py", "R": "R"} for k in list(lang_opts): if lang in k: lang = lang_opts[k] break # Obtain the specified script file. script = cmd + "." + lang if args.debug: print(DEBUG + "execute the script: " + os.path.join(path, script)) if cmd not in list(desc['commands']) or not os.path.exists(os.path.join(path, script)): msg = """{}The command '{}' was not found for this model. Try using 'commands' to list all supported commands: $ {} commands {} """.format(APPX, cmd, CMD, model) print(msg, file=sys.stderr) sys.exit(1) # Determine the interpreter to use # # .R => Rscript; .py => python, etc. interpreter = utils.interpreter(script) command = "export CMD_CWD='{}'; {} {} {}".format(os.getcwd(), interpreter, script, param) if args.debug: print(DEBUG + "(cd " + path + "; " + command + ")") proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() if proc.returncode != 0: print("An error was encountered:\n") print(errors.decode("utf-8")) else: # Suggest next step if not args.quiet: utils.print_next_step(cmd, description=desc, model=model)
def dispatch(args): """Dispatch other commands to the appropriate model provided script.""" cmd = args.cmd model = args.model path = utils.get_package_dir(model) param = " ".join(args.param) # Get working dir if any. if args.workding_dir is not None: utils.update_working_dir(model, args.workding_dir) if args.workding_dir == '': args.workding_dir = None else: args.working_dir = utils.get_working_dir(model) # Get conda environment name if any. conda_env_name = utils.get_conda_env_name(model) # Check that the model is installed and has commands. utils.check_model_installed(model) entry = utils.load_description(model) if 'commands' not in entry or len(entry['commands']) == 0: raise utils.CommandNotFoundException(cmd, model) # Correct misspelled command if possible. matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands'])) if matched_cmd is not None: cmd = matched_cmd # Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml. meta = entry['meta'] if 'display' in meta and cmd in meta['display'] and os.environ.get( 'DISPLAY', '') == '': msg = "Graphic display is required but not available for command '{}'. Continue" yes = utils.yes_or_no(msg, cmd, yes=False) if not yes: msg = """ To enable DISPLAY be sure to connect to the server using 'ssh -X' or else connect to the server's desktop using a local X server like X2Go. """ sys.stdout.write(msg) sys.exit(1) # Obtain the default/chosen language for the package. lang = meta["languages"] # Deal with malformed 'languages' field lang_opts = {"python": "py", "R": "R"} for k in list(lang_opts): if lang in k: lang = lang_opts[k] break # Obtain the specified script file. script = cmd + "." + lang logger = logging.getLogger(__name__) logger.debug("Execute the script: " + os.path.join(path, script)) if cmd not in list(entry['commands']) or not os.path.exists( os.path.join(path, script)): raise utils.CommandNotFoundException(cmd, model) # Determine the interpreter to use # # .R => Rscript; .py => python, etc. interpreter = utils.interpreter(script) # Change working dir if needed if args.workding_dir is not None: script = os.path.join(path, script) path = args.workding_dir # _MLHUB_CMD_CWD: a environment variable indicates current working # directory where command `ml xxx` is invoked. # _MLHUB_MODEL_NAME: env variable indicates the name of the model. # # The above two env vars can be obtained by helper function, such # as utils.get_cmd_cwd(). And model package developer should be # use the helper function instead of the env vars directly. command = "export _MLHUB_CMD_CWD='{}'; export _MLHUB_MODEL_NAME='{}'; {} {} {}".format( os.getcwd(), model, interpreter, script, param) # Run script inside conda environment if specified if conda_env_name is not None: command = 'bash -c "source activate {}; {}"'.format( conda_env_name, command) logger.debug("(cd " + path + "; " + command + ")") proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() missing_r_dep = False if proc.returncode != 0: errors = errors.decode("utf-8") # Check if it is Python dependency unsatisfied dep_required = re.compile( r"ModuleNotFoundError: No module named '(.*)'").search(errors) # Check if R dependency unsatisified if dep_required is None: dep_required = re.compile( r"there is no package called ‘(.*)’").search(errors) if dep_required is not None: missing_r_dep = True # Check if required data resource not found data_required = re.compile( r"mlhub.utils.DataResourceNotFoundException").search(errors) if dep_required is not None: # Dependency unsatisfied dep_required = dep_required.group(1) logger.error("Dependency unsatisfied: {}\n{}".format( dep_required, errors)) raise utils.LackDependencyException(dep_required, missing_r_dep, model) elif data_required is not None: # Data not found raise utils.DataResourceNotFoundException() else: # Other unknown errors print("An error was encountered:\n") print(errors) else: # Suggest next step if not args.quiet: utils.print_next_step(cmd, description=entry, model=model)
def dispatch(args): """Dispatch other commands to the appropriate model provided script.""" cmd = args.cmd model = args.model path = utils.get_package_dir(model) param = " ".join(args.param) # Get working dir if any. if args.workding_dir is not None: utils.update_working_dir(model, args.workding_dir) if args.workding_dir == '': args.workding_dir = None else: args.working_dir = utils.get_working_dir(model) # Get conda environment name if any. conda_env_name = utils.get_conda_env_name(model) # Check that the model is installed and has commands. utils.check_model_installed(model) entry = utils.load_description(model) if 'commands' not in entry or len(entry['commands']) == 0: raise utils.CommandNotFoundException(cmd, model) # Correct misspelled command if possible. matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands'])) if matched_cmd is not None: cmd = matched_cmd # Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml. meta = entry['meta'] if 'display' in meta and cmd in meta['display'] and os.environ.get( 'DISPLAY', '') == '': msg = "Graphic display is required but not available for command '{}'. Continue" yes = utils.yes_or_no(msg, cmd, yes=False) if not yes: msg = """ To enable DISPLAY be sure to connect to the server using 'ssh -X' or else connect to the server's desktop using a local X server like X2Go. """ sys.stdout.write(msg) sys.exit(1) # Obtain the default/chosen language for the package. lang = meta["languages"] # Deal with malformed 'languages' field lang_opts = {"python": "py", "R": "R"} for k in list(lang_opts): if lang in k: lang = lang_opts[k] break # Obtain the specified script file. script = cmd + "." + lang logger = logging.getLogger(__name__) logger.debug("Execute the script: " + os.path.join(path, script)) if cmd not in list(entry['commands']) or not os.path.exists( os.path.join(path, script)): raise utils.CommandNotFoundException(cmd, model) # Determine the interpreter to use # # .R => Rscript; .py => python, etc. interpreter = utils.interpreter(script) # Change working dir if needed if args.workding_dir is not None: script = os.path.join(path, script) path = args.workding_dir # Handle python environment python_pkg_bin = None python_pkg_path = None if script.endswith('py'): python_pkg_base = os.path.sep.join( [utils.get_package_dir(model), '.python']) python_pkg_path = python_pkg_base + site.USER_SITE python_pkg_bin = python_pkg_base + site.USER_BASE + '/bin' # TODO: Make sure to document: # $ sudo apt-get install -y python3-pip # $ /usr/bin/pip3 install mlhub # Since in DSVM, the default pip is conda's pip, so if we stick to # use system's command, then the installation of MLHub itself should # be completed via system's pip, otherwise, MLHub will not work. if sys.executable != SYS_PYTHON_CMD: python_pkg_path = python_pkg_base + site.getsitepackages()[0] python_pkg_bin = python_pkg_base + site.PREFIXES[0] + '/bin' if utils.get_sys_python_pkg_usage(model): utils.print_on_stderr(MSG_INCOMPATIBLE_PYTHON_ENV, model) # _MLHUB_CMD_CWD: a environment variable indicates current working # directory where command `ml xxx` is invoked. # _MLHUB_MODEL_NAME: env variable indicates the name of the model. # # The above two env vars can be obtained by helper function, such # as utils.get_cmd_cwd(). And model package developer should be # use the helper function instead of the env vars directly. env_var = "export _MLHUB_CMD_CWD='{}'; ".format(os.getcwd()) env_var += "export _MLHUB_MODEL_NAME='{}'; ".format(model) env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable) env_var += "export PYTHONPATH='{}'; ".format( python_pkg_path) if python_pkg_path else "" env_var += "export PATH=\"{}:$PATH\"; ".format( python_pkg_bin) if python_pkg_bin else "" command = "{}{} {} {}".format(env_var, interpreter, script, param) # Run script inside conda environment if specified if conda_env_name is not None: command = '{} -c "source activate {}; {}"'.format( BASH_CMD, conda_env_name, command) logger.debug("(cd " + path + "; " + command + ")") proc = subprocess.Popen(command, shell=True, cwd=path, stderr=subprocess.PIPE) output, errors = proc.communicate() missing_r_dep = False if proc.returncode != 0: errors = errors.decode("utf-8") # Check if it is Python dependency unsatisfied dep_required = re.compile( r"ModuleNotFoundError: No module named '(.*)'").search(errors) # Check if R dependency unsatisified if dep_required is None: dep_required = re.compile( r"there is no package called ‘(.*)’").search(errors) if dep_required is not None: missing_r_dep = True # Check if required data resource not found data_required = re.compile( r"mlhub.utils.DataResourceNotFoundException").search(errors) if dep_required is not None: # Dependency unsatisfied dep_required = dep_required.group(1) logger.error("Dependency unsatisfied: {}\n{}".format( dep_required, errors)) raise utils.LackDependencyException(dep_required, missing_r_dep, model) elif data_required is not None: # Data not found raise utils.DataResourceNotFoundException() else: # Other unknown errors print("An error was encountered:\n") print(errors) else: # Suggest next step if not args.quiet: utils.print_next_step(cmd, description=entry, model=model)