def install_packages(env, to_install=None, packages=None): if shared._is_anaconda(env): conda_bin = shared._conda_cmd(env) if hasattr(env, "conda_yaml"): Config = collections.namedtuple("Config", "base dist") config_file = Config(base=env.conda_yaml, dist=None) else: config_file = get_config_file(env, "packages-conda.yaml") if config_file.base is None and packages is None: packages = [] else: if to_install: (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) with open(config_file.base) as in_handle: channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])]) conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals()))) # install our customized packages if len(packages) > 0: pkgs_str = " ".join(packages) env.safe_run("{conda_bin} install --quiet -y {channels} {pkgs_str}".format(**locals())) for package in packages: _link_bin(package, env, conda_info, conda_bin) # work around ncurses issues -- we don't always get the R version # https://github.com/bioconda/bioconda-recipes/issues/637 env.safe_run("{conda_bin} update -y -c r ncurses".format(**locals())) for pkg in ["python", "conda", "pip"]: _link_bin(pkg, env, conda_info, conda_bin, [pkg], "bcbio_")
def install_packages(env, to_install=None, packages=None): """Install packages using the home brew package manager. Handles upgrading brew, tapping required repositories and installing or upgrading packages as appropriate. `to_install` is a CloudBioLinux compatible set of top level items to add, alternatively `packages` is a list of raw package names. """ config_file = get_config_file(env, "packages-homebrew.yaml") if to_install: (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) brew_cmd = _brew_cmd(env) formula_repos = ["homebrew/science"] env.safe_run("%s update" % brew_cmd) current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()]) for repo in formula_repos: if repo not in current_taps: env.safe_run("%s tap %s" % (brew_cmd, repo)) current_pkgs = set([x.strip() for x in env.safe_run_output("%s list" % brew_cmd).split()]) outdated_pkgs = set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]) for pkg in packages: if pkg in outdated_pkgs: brew_subcmd = "upgrade" elif pkg in current_pkgs: brew_subcmd = None else: brew_subcmd = "install" if brew_subcmd: env.safe_run("%s %s %s" % (brew_cmd, brew_subcmd, pkg))
def install_packages(env, to_install=None, packages=None): if shared._is_anaconda(env): conda_bin = shared._conda_cmd(env) config_file = get_config_file(env, "packages-conda.yaml") if config_file.base is None and packages is None: packages = [] else: if to_install: (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) with open(config_file.base) as in_handle: channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])]) conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals()))) if len(packages) > 0: pkgs_str = " ".join(packages) env.safe_run("{conda_bin} install -y {channels} {pkgs_str}".format(**locals())) for package in packages: _symlink_bin(package, env, conda_info, conda_bin) for pkg in ["python", "conda", "pip"]: _symlink_bin(pkg, env, conda_info, conda_bin, [pkg], "bcbio_") # remove packages that can cause failures # curl https://github.com/ContinuumIO/anaconda-issues/issues/72 problem_packages = ["curl"] pkgs_str = " ".join(problem_packages) with settings(warn_only=True): env.safe_run("{conda_bin} uninstall -y {pkgs_str}".format(**locals()))
def install_packages(env, to_install=None, packages=None): """Install packages using the home brew package manager. Handles upgrading brew, tapping required repositories and installing or upgrading packages as appropriate. `to_install` is a CloudBioLinux compatible set of top level items to add, alternatively `packages` is a list of raw package names. """ config_file = get_config_file(env, "packages-homebrew.yaml") if to_install: (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) # if we have no packages to install, do not try to install or update brew if len(packages) == 0: return system.install_homebrew(env) brew_cmd = _brew_cmd(env) formula_repos = ["homebrew/science", "chapmanb/cbl"] current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()]) _safe_update(env, brew_cmd, formula_repos, current_taps) for repo in formula_repos: if repo not in current_taps: env.safe_run("%s tap %s" % (brew_cmd, repo)) env.safe_run("%s tap --repair" % brew_cmd) ipkgs = {"outdated": set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]), "current": _get_current_pkgs(env, brew_cmd)} _install_brew_baseline(env, brew_cmd, ipkgs, packages) for pkg_str in packages: _install_pkg(env, pkg_str, brew_cmd, ipkgs)
def _setup_env(env): """ Setup the system environment required to run CloudMan. This means installing required system-level packages (as defined in CBL's ``packages.yaml``, or a flavor thereof) and Python dependencies (i.e., libraries) as defined in CloudMan's ``requirements.txt`` file. """ # Get and install required system packages if env.distribution in ["debian", "ubuntu"]: config_file = get_config_file(env, "packages.yaml") (packages, _) = _yaml_to_packages(config_file.base, 'cloudman') # Allow flavors to modify the package list packages = env.flavor.rewrite_config_items("packages", packages) _setup_apt_automation() _apt_packages(pkg_list=packages) elif env.distribution in ["centos", "scientificlinux"]: env.logger.warn("No CloudMan system package dependencies for CentOS") pass # Get and install required Python libraries with _make_tmp_dir() as work_dir: with cd(work_dir): url = os.path.join(CM_REPO_ROOT_URL, 'requirements.txt') _create_python_virtualenv(env, 'CM', reqs_url=url) # Add a custom vimrc vimrc_url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', 'vimrc') remote_file = '/etc/vim/vimrc' if env.safe_exists("/etc/vim"): env.safe_sudo("wget --output-document=%s %s" % (remote_file, vimrc_url)) env.logger.debug("Added a custom vimrc to {0}".format(remote_file)) # Setup profile aliases = ['alias lt="ls -ltr"', 'alias ll="ls -l"'] for alias in aliases: _add_to_profiles(alias, ['/etc/bash.bashrc']) env.logger.info("Done setting up CloudMan's environment")
def _setup_env(env): """ Setup the system environment required to run CloudMan. This primarily refers to installing required Python dependencies (ie, libraries) as defined in CloudMan's requirements.txt file. """ # Get and install required system packages if env.distribution in ["debian", "ubuntu"]: conf_file = 'config.yaml' url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', conf_file) cf = urllib.urlretrieve(url) (packages, _) = _yaml_to_packages(cf[0], 'cloudman') _apt_packages(pkg_list=packages) elif env.distibution in ["centos", "scientificlinux"]: env.logger.warn("No CloudMan system package dependencies for CentOS") pass reqs_file = 'requirements.txt' with _make_tmp_dir() as work_dir: with cd(work_dir): # Get and install requried Python libraries url = os.path.join(CM_REPO_ROOT_URL, reqs_file) run("wget --output-document=%s %s" % (reqs_file, url)) sudo("pip install --upgrade --requirement={0}".format(reqs_file)) # Add a custom vimrc vimrc_url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', 'vimrc') remote_file = '/etc/vim/vimrc' sudo("wget --output-document=%s %s" % (remote_file, vimrc_url)) env.logger.debug("Added a custom vimrc to {0}".format(remote_file)) env.logger.debug("Done setting up CloudMan's environment")
def install_custom(p, automated=False, pkg_to_group=None): """Install a single custom package by name. This method fetches names from custom.yaml that delegate to a method in the custom/name.py program. fab install_custom_package:package_name """ _setup_logging(env) env.logger.info("Install custom software packages") if not automated: _parse_fabricrc() _setup_edition(env) _setup_distribution_environment() _create_local_paths() pkg_config = os.path.join(env.config_dir, "custom.yaml") packages, pkg_to_group = _yaml_to_packages(pkg_config, None) try: env.logger.debug("Import %s" % p) mod = __import__("cloudbio.custom.%s" % pkg_to_group[p], fromlist=["cloudbio", "custom"]) except ImportError: raise ImportError("Need to write a %s module in custom." % pkg_to_group[p]) replace_chars = ["-"] try: for to_replace in replace_chars: p = p.replace(to_replace, "_") fn = getattr(mod, "install_%s" % p) except AttributeError: raise ImportError("Need to write a install_%s function in custom.%s" % (p, pkg_to_group[p])) fn(env)
def _custom_installs(to_install): if not exists(env.local_install): run("mkdir -p %s" % env.local_install) pkg_config = os.path.join(env.config_dir, "custom.yaml") packages, pkg_to_group = _yaml_to_packages(pkg_config, to_install) for p in env.flavor.rewrite_config_items("custom", packages): install_custom(p, True, pkg_to_group)
def _apt_packages(to_install): """Install packages available via apt-get. """ env.logger.info("Update and install all packages") pkg_config_file = os.path.join(env.config_dir, "packages.yaml") subs_pkg_config_file = os.path.join(env.config_dir, "packages-%s.yaml" % env.distribution) if not os.path.exists(subs_pkg_config_file): subs_pkg_config_file = None sudo("apt-get update") # Always update env.edition.apt_upgrade_system() # Retrieve final package names (packages, _) = _yaml_to_packages(pkg_config_file, to_install, subs_pkg_config_file) # At this point allow the Edition to rewrite the package list - # this is shared within and between editions. # Ref: https://github.com/chapmanb/cloudbiolinux/pull/10#issuecomment-1616423 packages = env.edition.rewrite_config_items("packages", packages) # At this point allow the Flavor to rewrite the package list packages = env.flavor.rewrite_config_items("packages", packages) # A single line install is much faster - note that there is a max # for the command line size, so we do 30 at a time group_size = 30 i = 0 env.logger.info("Updating %i packages" % len(packages)) while i < len(packages): sudo("apt-get -y --force-yes install %s" % " ".join(packages[i:i+group_size])) i += group_size sudo("apt-get clean")
def install_packages(env, to_install=None, packages=None): if shared._is_anaconda(env): conda_bin = shared._conda_cmd(env) config_file = get_config_file(env, "packages-conda.yaml") if config_file.base is None and packages is None: packages = [] else: if to_install: (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) with open(config_file.base) as in_handle: channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])]) conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals()))) # Transition change -- ensure installed perl is perl-threaded system_packages = ["perl", "perl-threaded"] pkgs_str = " ".join(system_packages) with settings(warn_only=True): env.safe_run("{conda_bin} uninstall -y {pkgs_str}".format(**locals())) # install our customized packages if len(packages) > 0: pkgs_str = " ".join(packages) env.safe_run("{conda_bin} install -y {channels} {pkgs_str}".format(**locals())) for package in packages: _link_bin(package, env, conda_info, conda_bin) for pkg in ["python", "conda", "pip"]: _link_bin(pkg, env, conda_info, conda_bin, [pkg], "bcbio_") # remove packages we want the system to supply # curl https://github.com/ContinuumIO/anaconda-issues/issues/72 system_packages = ["curl"] pkgs_str = " ".join(system_packages) with settings(warn_only=True): env.safe_run("{conda_bin} uninstall -y {pkgs_str}".format(**locals()))
def _custom_installs(to_install, ignore=None): if not env.safe_exists(env.local_install) and env.local_install: env.safe_run("mkdir -p %s" % env.local_install) pkg_config = get_config_file(env, "custom.yaml").base packages, pkg_to_group = _yaml_to_packages(pkg_config, to_install) packages = [p for p in packages if ignore is None or p not in ignore] for p in env.flavor.rewrite_config_items("custom", packages): install_custom(p, True, pkg_to_group)
def _apt_packages(to_install=None, pkg_list=None, pkg_config_file_path=None): """ Install packages available via apt-get. Note that ``to_install`` and ``pkg_list`` arguments cannot be used simultaneously. :type to_install: list :param to_install: A list of strings (ie, groups) present in the ``main.yaml`` config file that will be used to filter out the specific packages to be installed. :type pkg_list: list :param pkg_list: An explicit list of packages to install. No other files, flavors, or editions are considered. :type pkg_config_file_path: string :param pkg_config_file_path: Allows a custom path to be specified where ``packages.yaml`` and ``packages-[dist].yaml`` are stored. Note that the file names cannot be customized, only the path. """ env.logger.info("Update the system") sudo("apt-get update") # Always update if to_install is not None: env.logger.info("Will install all packages (as listed in packages*.yaml " "files in {0})".format(pkg_config_file_path)) if pkg_config_file_path is None: pkg_config_file_path = env.config_dir pkg_config_file = os.path.join(pkg_config_file_path, "packages.yaml") subs_pkg_config_file = os.path.join(pkg_config_file_path, "packages-%s.yaml" % env.distribution) if not os.path.exists(subs_pkg_config_file): subs_pkg_config_file = None env.edition.apt_upgrade_system() # Retrieve final package names (packages, _) = _yaml_to_packages(pkg_config_file, to_install, subs_pkg_config_file) # At this point allow the Edition to rewrite the package list - # this is shared within and between editions. # Ref: https://github.com/chapmanb/cloudbiolinux/pull/10#issuecomment-1616423 packages = env.edition.rewrite_config_items("packages", packages) # At this point allow the Flavor to rewrite the package list packages = env.flavor.rewrite_config_items("packages", packages) elif pkg_list is not None: env.logger.info("Will install specific packages: {0}".format(pkg_list)) packages = pkg_list else: raise ValueError("Need a file with packages or a list of packages") # A single line install is much faster - note that there is a max # for the command line size, so we do 30 at a time group_size = 30 i = 0 env.logger.info("Installing %i packages" % len(packages)) while i < len(packages): env.logger.info("Package install progress: {0}/{1}".format(i, len(packages))) sudo("apt-get -y --force-yes install %s" % " ".join(packages[i:i+group_size])) i += group_size sudo("apt-get clean")
def install_packages(env): config_file = get_config_file(env, "perl-libs.yaml") (packages, _) = _yaml_to_packages(config_file.base, subs_yaml_file=config_file.dist, namesort=False) cpanm_cmd = find_cmd(env, "cpanm", "--version") for package in packages: if package.count("==") > 1: _install_from_url(env, cpanm_cmd, package) else: _install_from_cpan(env, cpanm_cmd, package)
def _provision_puppet_classes(to_install, ignore=None): """ Much like _custom_installs, read config file, determine what to install, and install it. """ pkg_config = get_config_file(env, "puppet_classes.yaml").base packages, _ = _yaml_to_packages(pkg_config, to_install) packages = [p for p in packages if ignore is None or p not in ignore] classes = [recipe for recipe in env.flavor.rewrite_config_items("puppet_classes", packages)]
def install_packages(env, to_install=None, packages=None): if shared._is_anaconda(env): conda_bin = shared._conda_cmd(env) config_file = get_config_file(env, "packages-conda.yaml") if to_install: (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) with open(config_file.base) as in_handle: channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])]) if len(packages) > 0: for pkg in packages: env.safe_run("{conda_bin} install -y {channels} {pkg}".format(**locals()))
def _provision_chef_recipes(to_install, ignore=None): """ Much like _custom_installs, read config file, determine what to install, and install it. """ pkg_config = get_config_file(env, "chef_recipes.yaml").base packages, _ = _yaml_to_packages(pkg_config, to_install) packages = [p for p in packages if ignore is None or p not in ignore] recipes = [recipe for recipe in env.flavor.rewrite_config_items("chef_recipes", packages)] if recipes: # Don't bother running chef if nothing to configure install_chef_recipe(recipes, True)
def _nix_packages(to_install): """Install packages available via nixpkgs (optional) """ if env.nixpkgs: env.logger.info("Update and install NixPkgs packages") pkg_config_file = get_config_file(env, "packages-nix.yaml").base sudo("nix-channel --update") # Retrieve final package names (packages, _) = _yaml_to_packages(pkg_config_file, to_install) packages = env.flavor.rewrite_config_items("packages", packages) for p in packages: sudo("nix-env -b -i %s" % p)
def _custom_installs(to_install, ignore=None, add=None): if not env.safe_exists(env.local_install) and env.local_install: env.safe_run("mkdir -p %s" % env.local_install) pkg_config = get_config_file(env, "custom.yaml").base packages, pkg_to_group = _yaml_to_packages(pkg_config, to_install) packages = [p for p in packages if ignore is None or p not in ignore] if add is not None: for key, vals in add.iteritems(): for v in vals: pkg_to_group[v] = key packages.append(v) for p in env.flavor.rewrite_config_items("custom", packages): install_custom(p, True, pkg_to_group)
def _yum_packages(to_install): """Install rpm packages available via yum. """ pkg_config = os.path.join(env.config_dir, "packages-yum.yaml") with settings(warn_only=True): sudo("yum check-update") sudo("yum -y upgrade") # Retrieve packages to get and install each of them (packages, _) = _yaml_to_packages(pkg_config, to_install) # At this point allow the Flavor to rewrite the package list packages = env.flavor.rewrite_config_items("packages", packages) for package in packages: sudo("yum -y install %s" % package)
def install_custom(p, automated=False, pkg_to_group=None, flavor=None): """ Install a single custom program or package by name. This method fetches program name from ``config/custom.yaml`` and delegates to a method in ``custom/*name*.py`` to proceed with the installation. Alternatively, if a program install method is defined in the appropriate package, it will be called directly (see param ``p``). Usage: fab [-i key] [-u user] -H host install_custom:program_name :type p: string :param p: A name of the custom program to install. This has to be either a name that is listed in ``custom.yaml`` as a subordinate to a group name or a program name whose install method is defined in either ``cloudbio`` or ``custom`` packages (e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``). :type automated: bool :param automated: If set to True, the environment is not loaded and reading of the ``custom.yaml`` is skipped. """ _setup_logging(env) p = p.lower() # All packages listed in custom.yaml are in lower case time_start = _print_time_stats("Custom install for '{0}'".format(p), "start") if not automated: _configure_fabric_environment(env, flavor) pkg_config = get_config_file(env, "custom.yaml").base packages, pkg_to_group = _yaml_to_packages(pkg_config, None) try: env.logger.debug("Import %s" % p) # Allow direct calling of a program install method, even if the program # is not listed in the custom list (ie, not contained as a key value in # pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman. mod_name = pkg_to_group[p] if p in pkg_to_group else p mod = __import__("cloudbio.custom.%s" % mod_name, fromlist=["cloudbio", "custom"]) except ImportError: raise ImportError("Need to write a %s module in custom." % pkg_to_group[p]) replace_chars = ["-"] try: for to_replace in replace_chars: p = p.replace(to_replace, "_") fn = getattr(mod, "install_%s" % p) except AttributeError: raise ImportError("Need to write a install_%s function in custom.%s" % (p, pkg_to_group[p])) fn(env) _print_time_stats("Custom install for '%s'" % p, "end", time_start)
def install_custom(p, automated=False, pkg_to_group=None): """Install a single custom package by name. This method fetches names from custom.yaml that delegate to a method in the custom/name.py program. Alternatively, if a program install method is defined in approapriate package, it will be called directly (see param p). Usage: fab [-i key] [-u user] -H host install_custom:program_name :type p: string :param p: A name of a custom program to install. This has to be either a name that is listed in custom.yaml as a subordinate to a group name or a program name whose install method is defined in either cloudbio or custom packages (eg, install_cloudman). :type automated: bool :param automated: If set to True, the environment is not loaded and reading of the custom.yaml is skipped. """ _setup_logging(env) p = p.lower() # All packages are listed in custom.yaml are in lower case time_start = _print_time_stats("Custom install for '{0}'".format(p), "start") if not automated: _parse_fabricrc() _setup_edition(env) _setup_distribution_environment() _create_local_paths() pkg_config = os.path.join(env.config_dir, "custom.yaml") packages, pkg_to_group = _yaml_to_packages(pkg_config, None) try: env.logger.debug("Import %s" % p) # Allow direct calling of a program install method, even if the program # is not listed in the custom list (ie, not contained as a key value in # pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman. mod_name = pkg_to_group[p] if p in pkg_to_group else p mod = __import__("cloudbio.custom.%s" % mod_name, fromlist=["cloudbio", "custom"]) except ImportError: raise ImportError("Need to write a %s module in custom." % pkg_to_group[p]) replace_chars = ["-"] try: for to_replace in replace_chars: p = p.replace(to_replace, "_") fn = getattr(mod, "install_%s" % p) except AttributeError: raise ImportError("Need to write a install_%s function in custom.%s" % (p, pkg_to_group[p])) fn(env) _print_time_stats("Custom install for '%s'" % p, "end", time_start)
def _yum_packages(to_install): """Install rpm packages available via yum. """ if env.distribution == "scientificlinux": package_file = "packages-scientificlinux.yaml" else: package_file = "packages-yum.yaml" pkg_config = get_config_file(env, package_file).base with settings(warn_only=True): sudo("yum check-update") sudo("yum -y upgrade") # Retrieve packages to get and install each of them (packages, _) = _yaml_to_packages(pkg_config, to_install) # At this point allow the Flavor to rewrite the package list packages = env.flavor.rewrite_config_items("packages", packages) for package in packages: sudo("yum -y install %s" % package)
def install_packages(env, to_install=None, packages=None): if shared._is_anaconda(env): conda_bin = shared._conda_cmd(env) if hasattr(env, "conda_yaml"): Config = collections.namedtuple("Config", "base dist") config_file = Config(base=env.conda_yaml, dist=None) else: config_file = get_config_file(env, "packages-conda.yaml") if config_file.base is None and packages is None: packages = [] else: if to_install: (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) with open(config_file.base) as in_handle: channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])]) conda_envs = _create_environments(env, conda_bin, packages) conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals()))) # Uninstall old R packages that clash with updated versions # Temporary fix to allow upgrades from older versions that have migrated # r-tximport is now bioconductor-tximport # py2cairo is incompatible with r 3.4.1 # libedit pins to curses 6.0 but bioconda requires 5.9 for problem in ["r-tximport", "py2cairo", "libedit"]: cur_packages = [x["name"] for x in json.loads(env.safe_run_output("{conda_bin} list --json {problem}".format(**locals())))] if problem in cur_packages: env.safe_run("{conda_bin} remove --force -y {problem}".format(**locals())) # install our customized packages if len(packages) > 0: for env_name, env_packages in _split_by_condaenv(packages): if env_name: assert env_name in conda_envs, (env_name, conda_envs) env_str = "-n %s" % env_name else: env_str = "" pkgs_str = " ".join(env_packages) env.safe_run("{conda_bin} install --quiet -y {env_str} {channels} {pkgs_str}".format(**locals())) conda_pkg_list = json.loads(env.safe_run_output( "{conda_bin} list --json {env_str}".format(**locals()))) for package in env_packages: _link_bin(package, env, conda_info, conda_bin, conda_pkg_list, conda_envdir=conda_envs.get(env_name)) conda_pkg_list = json.loads(env.safe_run_output("{conda_bin} list --json".format(**locals()))) for pkg in ["python", "conda", "pip"]: _link_bin(pkg, env, conda_info, conda_bin, conda_pkg_list, files=[pkg], prefix="bcbio_")
def _apt_packages(to_install=None, pkg_list=None): """ Install packages available via apt-get. Note that ``to_install`` and ``pkg_list`` arguments cannot be used simultaneously. :type to_install: list :param to_install: A list of strings (ie, groups) present in the ``main.yaml`` config file that will be used to filter out the specific packages to be installed. :type pkg_list: list :param pkg_list: An explicit list of packages to install. No other files, flavors, or editions are considered. """ if env.edition.short_name not in ["minimal"]: env.logger.info("Update the system") with settings(warn_only=True): sudo("apt-get update") if to_install is not None: config_file = get_config_file(env, "packages.yaml") env.edition.apt_upgrade_system() (packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist) # Allow editions and flavors to modify the package list packages = env.edition.rewrite_config_items("packages", packages) packages = env.flavor.rewrite_config_items("packages", packages) elif pkg_list is not None: env.logger.info("Will install specific packages: {0}".format(pkg_list)) packages = pkg_list else: raise ValueError("Need a file with packages or a list of packages") # A single line install is much faster - note that there is a max # for the command line size, so we do 30 at a time group_size = 30 i = 0 env.logger.info("Installing %i packages" % len(packages)) while i < len(packages): env.logger.info("Package install progress: {0}/{1}".format( i, len(packages))) sudo("apt-get -y --force-yes install %s" % " ".join( packages[i:i + group_size])) i += group_size sudo("apt-get clean")
def install_custom(p, automated=False, pkg_to_group=None, flavor=None): """ Install a single custom program or package by name. This method fetches program name from ``config/custom.yaml`` and delegates to a method in ``custom/*name*.py`` to proceed with the installation. Alternatively, if a program install method is defined in the appropriate package, it will be called directly (see param ``p``). Usage: fab [-i key] [-u user] -H host install_custom:program_name :type p: string :param p: A name of the custom program to install. This has to be either a name that is listed in ``custom.yaml`` as a subordinate to a group name or a program name whose install method is defined in either ``cloudbio`` or ``custom`` packages (e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``). :type automated: bool :param automated: If set to True, the environment is not loaded and reading of the ``custom.yaml`` is skipped. """ p = p.lower() # All packages listed in custom.yaml are in lower case if not automated: _setup_logging(env) _configure_fabric_environment(env, flavor, ignore_distcheck=True) pkg_config = get_config_file(env, "custom.yaml").base packages, pkg_to_group = _yaml_to_packages(pkg_config, None) time_start = _print_time_stats("Custom install for '{0}'".format(p), "start") fn = _custom_install_function(env, p, pkg_to_group) fn(env) ## TODO: Replace the previous 4 lines with the following one, barring ## objections. Slightly different behavior because pkg_to_group will be ## loaded regardless of automated if it is None, but IMO this shouldn't ## matter because the following steps look like they would fail if ## automated is True and pkg_to_group is None. # _install_custom(p, pkg_to_group) _print_time_stats("Custom install for '%s'" % p, "end", time_start)
def install_in(conda_bin, system_installdir, config_file=None, packages=None): """Install packages inside a given anaconda directory. New approach, local only and not dependent on fabric. """ if config_file is None and packages is None: packages = [] check_channels = [] else: (packages, _) = _yaml_to_packages(config_file) with open(config_file) as in_handle: check_channels = yaml.safe_load(in_handle).get("channels", []) channels = " ".join(["-c %s" % x for x in check_channels]) conda_envs = _create_environments(conda_bin, packages) for env_dir in conda_envs.values(): _clean_environment(env_dir) conda_info = json.loads(subprocess.check_output("{conda_bin} info --json".format(**locals()), shell=True)) # Uninstall old R packages that clash with updated versions # Temporary fix to allow upgrades from older versions that have migrated # r-tximport is now bioconductor-tximport # py2cairo is incompatible with r 3.4.1+ problems = ["r-tximport", "py2cairo"] # Add packages migrated into separate environments, like python2 for env_name, env_packages in _split_by_condaenv(packages): if env_name: problems += env_packages if problems: print("Checking for problematic or migrated packages in default environment") cur_packages = [x["name"] for x in json.loads(subprocess.check_output("%s list --json" % (conda_bin), shell=True)) if x["name"] in problems and x["channel"] in check_channels] if cur_packages: print("Found packages that moved from default environment: %s" % ", ".join(cur_packages)) problems = " ".join(cur_packages) subprocess.check_call("{conda_bin} remove -y {problems}".format(**locals()), shell=True) # install our customized packages if len(packages) > 0: for env_name, env_packages in _split_by_condaenv(packages): print("# Installing into conda environment %s: %s" % (env_name or "default", ", ".join(env_packages))) if env_name: assert env_name in conda_envs, (env_name, conda_envs) env_str = "-n %s" % env_name else: env_str = "" pkgs_str = " ".join(["'%s'" % x for x in sorted(env_packages)]) py_version = ENV_PY_VERSIONS[env_name] # During openssl transition, pin to 1.1.1 to avoid slow resolve times extra_pins = "'openssl=1.1.1b'" if "deepvariant" in env_packages: # Ignore /etc/boto.cfg which creates conflicts with conda gsutils # https://github.com/GoogleCloudPlatform/gsutil/issues/516 exports = "export BOTO_CONFIG=/ignoreglobal && " else: exports = "" subprocess.check_call("{exports}{conda_bin} install -y {env_str} {channels} " "{py_version} {extra_pins} {pkgs_str}".format(**locals()), shell=True) conda_pkg_list = json.loads(subprocess.check_output( "{conda_bin} list --json {env_str}".format(**locals()), shell=True)) for package in env_packages: _link_bin(package, system_installdir, conda_info, conda_bin, conda_pkg_list, conda_envdir=conda_envs.get(env_name)) conda_pkg_list = json.loads(subprocess.check_output("{conda_bin} list --json".format(**locals()), shell=True)) for pkg in ["python", "conda", "pip"]: _link_bin(pkg, system_installdir, conda_info, conda_bin, conda_pkg_list, files=[pkg], prefix="bcbio_")
def _install_custom(p, pkg_to_group=None): if pkg_to_group is None: pkg_config = get_config_file(env, "custom.yaml").base packages, pkg_to_group = _yaml_to_packages(pkg_config, None) fn = _custom_install_function(env, p, pkg_to_group) fn(env)