Example #1
0
def dispatch(args):
    """Dispatch other commands to the appropriate model provided script."""

    cmd = args.cmd
    model = args.model
    path = utils.get_package_dir(model)

    param = " ".join(args.param)

    # Get working dir if any.

    if args.workding_dir is not None:
        utils.update_working_dir(model, args.workding_dir)
        if args.workding_dir == '':
            args.workding_dir = None
    else:
        args.working_dir = utils.get_working_dir(model)

    # Get conda environment name if any.

    conda_env_name = utils.get_conda_env_name(model)

    # Check that the model is installed and has commands.

    utils.check_model_installed(model)

    entry = utils.load_description(model)

    if 'commands' not in entry or len(entry['commands']) == 0:
        raise utils.CommandNotFoundException(cmd, model)

    # Correct misspelled command if possible.

    matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands']))
    if matched_cmd is not None:
        cmd = matched_cmd

    # Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.

    meta = entry['meta']
    if 'display' in meta and cmd in meta['display'] and os.environ.get(
            'DISPLAY', '') == '':
        msg = "Graphic display is required but not available for command '{}'. Continue"
        yes = utils.yes_or_no(msg, cmd, yes=False)
        if not yes:
            msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.

"""
            sys.stdout.write(msg)
            sys.exit(1)

    # Obtain the default/chosen language for the package.

    lang = meta["languages"]

    # Deal with malformed 'languages' field

    lang_opts = {"python": "py", "R": "R"}
    for k in list(lang_opts):
        if lang in k:
            lang = lang_opts[k]
            break

    # Obtain the specified script file.

    script = cmd + "." + lang

    logger = logging.getLogger(__name__)
    logger.debug("Execute the script: " + os.path.join(path, script))

    if cmd not in list(entry['commands']) or not os.path.exists(
            os.path.join(path, script)):
        raise utils.CommandNotFoundException(cmd, model)

    # Determine the interpreter to use
    #
    # .R => Rscript; .py => python, etc.

    interpreter = utils.interpreter(script)

    # Change working dir if needed

    if args.workding_dir is not None:
        script = os.path.join(path, script)
        path = args.workding_dir

    # _MLHUB_CMD_CWD: a environment variable indicates current working
    #                 directory where command `ml xxx` is invoked.
    # _MLHUB_MODEL_NAME: env variable indicates the name of the model.
    #
    # The above two env vars can be obtained by helper function, such
    # as utils.get_cmd_cwd().  And model package developer should be
    # use the helper function instead of the env vars directly.

    command = "export _MLHUB_CMD_CWD='{}'; export _MLHUB_MODEL_NAME='{}'; {} {} {}".format(
        os.getcwd(), model, interpreter, script, param)

    # Run script inside conda environment if specified

    if conda_env_name is not None:
        command = 'bash -c "source activate {}; {}"'.format(
            conda_env_name, command)

    logger.debug("(cd " + path + "; " + command + ")")

    proc = subprocess.Popen(command,
                            shell=True,
                            cwd=path,
                            stderr=subprocess.PIPE)
    output, errors = proc.communicate()
    missing_r_dep = False
    if proc.returncode != 0:
        errors = errors.decode("utf-8")

        # Check if it is Python dependency unsatisfied

        dep_required = re.compile(
            r"ModuleNotFoundError: No module named '(.*)'").search(errors)

        # Check if R dependency unsatisified

        if dep_required is None:
            dep_required = re.compile(
                r"there is no package called ‘(.*)’").search(errors)
            if dep_required is not None:
                missing_r_dep = True

        # Check if required data resource not found

        data_required = re.compile(
            r"mlhub.utils.DataResourceNotFoundException").search(errors)

        if dep_required is not None:  # Dependency unsatisfied
            dep_required = dep_required.group(1)
            logger.error("Dependency unsatisfied: {}\n{}".format(
                dep_required, errors))
            raise utils.LackDependencyException(dep_required, missing_r_dep,
                                                model)
        elif data_required is not None:  # Data not found
            raise utils.DataResourceNotFoundException()
        else:  # Other unknown errors
            print("An error was encountered:\n")
            print(errors)

    else:
        # Suggest next step

        if not args.quiet:
            utils.print_next_step(cmd, description=entry, model=model)
Example #2
0
def dispatch(args):
    """Dispatch other commands to the appropriate model provided script."""

    cmd = args.cmd
    model = args.model
    path = utils.get_package_dir(model)

    param = " ".join(args.param)

    # Get working dir if any.

    if args.workding_dir is not None:
        utils.update_working_dir(model, args.workding_dir)
        if args.workding_dir == '':
            args.workding_dir = None
    else:
        args.working_dir = utils.get_working_dir(model)

    # Get conda environment name if any.

    conda_env_name = utils.get_conda_env_name(model)

    # Check that the model is installed and has commands.

    utils.check_model_installed(model)

    entry = utils.load_description(model)

    if 'commands' not in entry or len(entry['commands']) == 0:
        raise utils.CommandNotFoundException(cmd, model)

    # Correct misspelled command if possible.

    matched_cmd = utils.get_misspelled_command(cmd, list(entry['commands']))
    if matched_cmd is not None:
        cmd = matched_cmd

    # Check if cmd needs to use graphic display indicated in DESCRIPTION.yaml.

    meta = entry['meta']
    if 'display' in meta and cmd in meta['display'] and os.environ.get(
            'DISPLAY', '') == '':
        msg = "Graphic display is required but not available for command '{}'. Continue"
        yes = utils.yes_or_no(msg, cmd, yes=False)
        if not yes:
            msg = """
To enable DISPLAY be sure to connect to the server using 'ssh -X'
or else connect to the server's desktop using a local X server like X2Go.

"""
            sys.stdout.write(msg)
            sys.exit(1)

    # Obtain the default/chosen language for the package.

    lang = meta["languages"]

    # Deal with malformed 'languages' field

    lang_opts = {"python": "py", "R": "R"}
    for k in list(lang_opts):
        if lang in k:
            lang = lang_opts[k]
            break

    # Obtain the specified script file.

    script = cmd + "." + lang

    logger = logging.getLogger(__name__)
    logger.debug("Execute the script: " + os.path.join(path, script))

    if cmd not in list(entry['commands']) or not os.path.exists(
            os.path.join(path, script)):
        raise utils.CommandNotFoundException(cmd, model)

    # Determine the interpreter to use
    #
    # .R => Rscript; .py => python, etc.

    interpreter = utils.interpreter(script)

    # Change working dir if needed

    if args.workding_dir is not None:
        script = os.path.join(path, script)
        path = args.workding_dir

    # Handle python environment

    python_pkg_bin = None
    python_pkg_path = None
    if script.endswith('py'):
        python_pkg_base = os.path.sep.join(
            [utils.get_package_dir(model), '.python'])
        python_pkg_path = python_pkg_base + site.USER_SITE
        python_pkg_bin = python_pkg_base + site.USER_BASE + '/bin'

        # TODO: Make sure to document:
        #     $ sudo apt-get install -y python3-pip
        #     $ /usr/bin/pip3 install mlhub
        #   Since in DSVM, the default pip is conda's pip, so if we stick to
        #   use system's command, then the installation of MLHub itself should
        #   be completed via system's pip, otherwise, MLHub will not work.

        if sys.executable != SYS_PYTHON_CMD:
            python_pkg_path = python_pkg_base + site.getsitepackages()[0]
            python_pkg_bin = python_pkg_base + site.PREFIXES[0] + '/bin'
            if utils.get_sys_python_pkg_usage(model):
                utils.print_on_stderr(MSG_INCOMPATIBLE_PYTHON_ENV, model)

    # _MLHUB_CMD_CWD: a environment variable indicates current working
    #                 directory where command `ml xxx` is invoked.
    # _MLHUB_MODEL_NAME: env variable indicates the name of the model.
    #
    # The above two env vars can be obtained by helper function, such
    # as utils.get_cmd_cwd().  And model package developer should be
    # use the helper function instead of the env vars directly.

    env_var = "export _MLHUB_CMD_CWD='{}'; ".format(os.getcwd())
    env_var += "export _MLHUB_MODEL_NAME='{}'; ".format(model)
    env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable)
    env_var += "export PYTHONPATH='{}'; ".format(
        python_pkg_path) if python_pkg_path else ""
    env_var += "export PATH=\"{}:$PATH\"; ".format(
        python_pkg_bin) if python_pkg_bin else ""

    command = "{}{} {} {}".format(env_var, interpreter, script, param)

    # Run script inside conda environment if specified

    if conda_env_name is not None:
        command = '{} -c "source activate {}; {}"'.format(
            BASH_CMD, conda_env_name, command)

    logger.debug("(cd " + path + "; " + command + ")")

    proc = subprocess.Popen(command,
                            shell=True,
                            cwd=path,
                            stderr=subprocess.PIPE)
    output, errors = proc.communicate()
    missing_r_dep = False
    if proc.returncode != 0:
        errors = errors.decode("utf-8")

        # Check if it is Python dependency unsatisfied

        dep_required = re.compile(
            r"ModuleNotFoundError: No module named '(.*)'").search(errors)

        # Check if R dependency unsatisified

        if dep_required is None:
            dep_required = re.compile(
                r"there is no package called ‘(.*)’").search(errors)
            if dep_required is not None:
                missing_r_dep = True

        # Check if required data resource not found

        data_required = re.compile(
            r"mlhub.utils.DataResourceNotFoundException").search(errors)

        if dep_required is not None:  # Dependency unsatisfied
            dep_required = dep_required.group(1)
            logger.error("Dependency unsatisfied: {}\n{}".format(
                dep_required, errors))
            raise utils.LackDependencyException(dep_required, missing_r_dep,
                                                model)
        elif data_required is not None:  # Data not found
            raise utils.DataResourceNotFoundException()
        else:  # Other unknown errors
            print("An error was encountered:\n")
            print(errors)

    else:
        # Suggest next step

        if not args.quiet:
            utils.print_next_step(cmd, description=entry, model=model)