Exemplo n.º 1
0
def configure_model(args):
    """Ensure the user's environment is configured."""

    # TODO: Add support for additional configuration if any except those
    #       specified in MLHUB.yaml.
    # TODO: When fail, print out the failed dep, as well as installed
    #       deps and non-installed deps.
    # TODO: Add support for specifying packages version.
    # TODO: Add more informative messages for different kinds of
    #       dependencies.

    # Other ideas for configuration
    #
    # 1 Construct mlhub container (from Ubuntu) with known starting point
    #
    # 2 Assume the user is on a DSVM with free Azure account to test out.
    #
    # 3 Read dependencies: and language: and then install as required:
    #
    # 4 Assume model packager provides a configure.R script. This is a
    #   override and no other configuration happens if this is
    #   supplied. Alternatively this is viewed as a cop-out prividing
    #   no support from mlhub for the model packager. The preference
    #   would be to use the dependencies: tag to list the required R
    #   or python packages.
    #
    # So the meta-code might be
    #
    #   if file.exists(configure.XX):
    #     XX configure.XX
    #   else if language: == "Rscript":
    #     packages <- dependencies:
    #     install  <- packages[!(packages %in% installed.packages()[,"Package"])]
    #     if(length(new.packages)) install.packages(install)
    #   else if language: == "python":
    #     packages = dependencies:
    #     cat pacakges > requirements.txt
    #     pip install -r requirements.txt
    #

    if not args.model:

        # Configure MLHUB per se.
        # Includes bash completion and system pre-requisites

        if distro.id() in ['debian', 'ubuntu']:
            path = os.path.dirname(__file__)
            command = '/bin/bash {}'.format(
                os.path.join('scripts', 'dep', 'mlhub.sh'))
            proc = subprocess.Popen(command,
                                    shell=True,
                                    cwd=path,
                                    stderr=subprocess.PIPE)
            output, errors = proc.communicate()
            if proc.returncode != 0:
                errors = errors.decode("utf-8")
                print("\nAn error was encountered:\n")
                print(errors)
                raise utils.ConfigureFailedException()

        return

    # Model package configuration.

    model = args.model

    # Correct model name if possible.

    matched_model = utils.get_misspelled_pkg(model)
    if matched_model is not None:
        model = matched_model

    # Setup.

    pkg_dir = utils.get_package_dir(model)

    # Check if the model package is installed.

    utils.check_model_installed(model)

    # Install dependencies specified in MLHUB.yaml

    entry = utils.load_description(model)
    depspec = None
    if 'dependencies' in entry:
        depspec = entry['dependencies']
    elif 'dependencies' in entry['meta']:
        depspec = entry['meta']['dependencies']

    if depspec is not None:
        for spec in utils.flatten_mlhubyaml_deps(depspec):
            category = spec[0][-1]
            deplist = spec[1]

            # Category include:
            #   ------------------------------------------------------------------------------
            #           category | action
            #   -----------------|------------------------------------------------------------
            #              None  |  install package according to entry['meta']['languages']
            #                    |  if R,      install.packages(xxx) from cran;
            #                    |  if Python, pip install xxx
            #   -----------------|------------------------------------------------------------
            #            system  |  apt-get install
            #                sh  |  apt-get install
            #   -----------------|------------------------------------------------------------
            #                 r  |  install.packages(xxx) from cran, version can be specified
            #              cran  |  install.packages(xxx) from cran, version can be specified
            #   cran-2018-12-01  |  install cran snapshot on 2018-12-01
            #            github  |  devtools::install_github from github
            #   -----------------|------------------------------------------------------------
            #            python  |  apt-get install python-xxx
            #           python3  |  apt-get install python3-xxx
            #               pip  |  pip install
            #              pip3  |  pip3 install
            #             conda  |  conda install
            #   -----------------|------------------------------------------------------------
            #             files  |  download files
            #   -----------------|------------------------------------------------------------

            # ----- Determine deps by language -----

            if category is None:

                lang = entry['meta']['languages'].lower()
                if lang == 'r':
                    utils.install_r_deps(deplist, model, source='cran')
                elif 'python'.startswith(lang):
                    utils.install_python_deps(deplist, model, source='pip')

            # ----- System deps -----

            elif category == 'system' or 'shell'.startswith(category):
                utils.install_system_deps(deplist)

            # ----- R deps -----

            elif category == 'r':
                utils.install_r_deps(deplist, model, source='cran')

            elif category == 'cran' or category == 'github' or category.startswith(
                    'cran-'):
                utils.install_r_deps(deplist, model, source=category)

            # ----- Python deps -----

            elif category.startswith('python') or category.startswith(
                    'pip') or category == 'conda':
                utils.install_python_deps(deplist, model, source=category)

            # ----- Files -----

            elif 'files'.startswith(category):
                utils.install_file_deps(deplist, model)

    # Run additional configure script if any.

    conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True
    conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf
    conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf

    if not conf:
        if depspec is not None:
            msg = ("No configuration script provided for this model. "
                   "The following dependencies are required:\n")
            print(msg)
            print(yaml.dump(depspec))
        else:
            print("No configuration provided (maybe none is required).")

    # Update working dir if any.

    if args.workding_dir is not None:
        utils.update_working_dir(model, args.workding_dir)

    # Suggest next step.

    if not args.quiet:
        utils.print_next_step('configure', model=model)
Exemplo n.º 2
0
def configure_model(args):
    """Ensure the user's environment is configured."""

    # TODO: Add support for additional configuration if any except those
    #       specified in MLHUB.yaml.
    # TODO: When fail, print out the failed dep, as well as installed
    #       deps and non-installed deps.
    # TODO: Add support for specifying packages version.
    # TODO: Add more informative messages for different kinds of
    #       dependencies.

    # Other ideas for configuration
    #
    # 1 Construct mlhub container (from Ubuntu) with known starting point
    #
    # 2 Assume the user is on a DSVM with free Azure account to test out.
    #
    # 3 Read dependencies: and language: and then install as required:
    #
    # 4 Assume model packager provides a configure.R script. This is a
    #   override and no other configuration happens if this is
    #   supplied. Alternatively this is viewed as a cop-out providing
    #   no support from mlhub for the model packager. The preference
    #   would be to use the dependencies: tag to list the required R
    #   or python packages.
    #
    # So the meta-code might be
    #
    #   if file.exists(configure.XX):
    #     XX configure.XX
    #   else if language: == "Rscript":
    #     packages <- dependencies:
    #     install  <- packages[!(packages %in% installed.packages()[,"Package"])]
    #     if(length(new.packages)) install.packages(install)
    #   else if language: == "python":
    #     packages = dependencies:
    #     cat packages > requirements.txt
    #     pip install -r requirements.txt
    #

    YES = args.y | args.yes

    # Avoid 403 errors which result when the header identifies itself
    # as python urllib or is empty and thus the web site assumes it is
    # a robot. We are not a robot but a user downloading a file. This
    # will ensure gitlab is okay with retrieving from a URL by adding
    # a header rather than no header. TODO move to using Requests.
    
    opener = urllib.request.build_opener()
    opener.addheaders = [('User-agent', 'Mozilla/5.0')]
    urllib.request.install_opener(opener)

    if not args.model:

        # Configure MLHUB per se.
        # Includes bash completion and system pre-requisites

        if distro.id() in ["debian", "ubuntu"]:
            path = os.path.dirname(__file__)
            env_var = "export _MLHUB_OPTION_YES='y'; " if YES else ""
            env_var += 'export _MLHUB_PYTHON_EXE="{}"; '.format(sys.executable)
            script = os.path.join("scripts", "dep", "mlhub.sh")
            command = "{}{} {}".format(env_var, BASH_CMD, script)
            proc = subprocess.Popen(
                command, shell=True, cwd=path, stderr=subprocess.PIPE
            )
            output, errors = proc.communicate()
            if proc.returncode != 0:
                raise utils.ConfigureFailedException(errors.decode("utf-8"))

        return

    # Model package configuration.

    model = args.model

    # Correct model name if possible.

    matched_model = utils.get_misspelled_pkg(model)
    if matched_model is not None:
        model = matched_model

    # Setup.

    pkg_dir = utils.get_package_dir(model)

    # Check if the model package is installed.

    utils.check_model_installed(model)

    # Install dependencies specified in MLHUB.yaml

    entry = utils.load_description(model)
    depspec = None
    if "dependencies" in entry:
        depspec = entry["dependencies"]
    elif "dependencies" in entry["meta"]:
        depspec = entry["meta"]["dependencies"]

    if depspec is not None:
        for spec in utils.flatten_mlhubyaml_deps(depspec):
            category = spec[0][-1]
            deplist = spec[1]

            # Category include:
            #   ------------------------------------------------------------------------------
            #           category | action
            #   -----------------|------------------------------------------------------------
            #              None  |  install package according to entry['meta']['languages']
            #                    |  if R,      install.packages(xxx) from cran;
            #                    |  if Python, pip install xxx
            #   -----------------|------------------------------------------------------------
            #            system  |  apt-get install
            #                sh  |  apt-get install
            #   -----------------|------------------------------------------------------------
            #                 r  |  install.packages(xxx) from cran, version can be specified
            #              cran  |  install.packages(xxx) from cran, version can be specified
            #   cran-2018-12-01  |  install cran snapshot on 2018-12-01
            #            github  |  devtools::install_github from github
            #   -----------------|------------------------------------------------------------
            #            python  |  apt-get install python-xxx
            #           python3  |  apt-get install python3-xxx
            #               pip  |  pip install
            #              pip3  |  pip3 install
            #             conda  |  conda install
            #   -----------------|------------------------------------------------------------
            #             files  |  download files
            #   -----------------|------------------------------------------------------------

            # ----- Determine deps by language -----

            if category is None:

                lang = entry["meta"]["languages"].lower()
                if lang == "r":
                    utils.install_r_deps(
                        deplist, model, source="cran", yes=YES
                    )
                elif "python".startswith(lang):
                    utils.install_python_deps(
                        deplist, model, source="pip", yes=YES
                    )

            # ----- System deps -----

            elif category == "system" or "shell".startswith(category):
                utils.install_system_deps(deplist, yes=YES)

            # ----- R deps -----

            elif category == "r":
                utils.install_r_deps(deplist, model, source="cran", yes=YES)

            elif (
                category == "cran"
                or category == "github"
                or category.startswith("cran-")
            ):
                utils.install_r_deps(deplist, model, source=category, yes=YES)

            # ----- Python deps -----

            elif (
                category.startswith("python")
                or category.startswith("pip")
                or category == "conda"
            ):
                utils.install_python_deps(
                    deplist, model, source=category, yes=YES
                )

            # ----- Files -----

            elif "files".startswith(category):
                utils.install_file_deps(deplist, model, key=args.i, yes=YES)

    # Run additional configure script if any.

    conf = utils.configure(pkg_dir, "configure.sh", args.quiet) or True
    conf = utils.configure(pkg_dir, "configure.R", args.quiet) or conf
    conf = utils.configure(pkg_dir, "configure.py", args.quiet) or conf

    if not conf:
        if depspec is not None:
            msg = (
                "No configuration script provided for this model. "
                "The following dependencies are required:\n"
            )
            print(msg)
            print(yaml.dump(depspec))
        else:
            print("No configuration provided (maybe none is required).")

    # Update working dir if any.

    if args.working_dir is not None:
        utils.update_working_dir(model, args.working_dir)

    # Suggest next step.

    if not args.quiet:
        utils.print_next_step("configure", model=model)