def install_packages(env, to_install=None, packages=None):
if shared._is_anaconda(env):
conda_bin = shared._conda_cmd(env)
if hasattr(env, "conda_yaml"):
Config = collections.namedtuple("Config", "base dist")
config_file = Config(base=env.conda_yaml, dist=None)
else:
config_file = get_config_file(env, "packages-conda.yaml")
if config_file.base is None and packages is None:
packages = []
else:
if to_install:
(packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
with open(config_file.base) as in_handle:
channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])])
conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals())))
# install our customized packages
if len(packages) > 0:
pkgs_str = " ".join(packages)
env.safe_run("{conda_bin} install --quiet -y {channels} {pkgs_str}".format(**locals()))
for package in packages:
_link_bin(package, env, conda_info, conda_bin)
# work around ncurses issues -- we don't always get the R version
# https://github.com/bioconda/bioconda-recipes/issues/637
env.safe_run("{conda_bin} update -y -c r ncurses".format(**locals()))
for pkg in ["python", "conda", "pip"]:
_link_bin(pkg, env, conda_info, conda_bin, [pkg], "bcbio_")
def install_packages(env, to_install=None, packages=None):
"""Install packages using the home brew package manager.
Handles upgrading brew, tapping required repositories and installing or upgrading
packages as appropriate.
`to_install` is a CloudBioLinux compatible set of top level items to add,
alternatively `packages` is a list of raw package names.
"""
config_file = get_config_file(env, "packages-homebrew.yaml")
if to_install:
(packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
brew_cmd = _brew_cmd(env)
formula_repos = ["homebrew/science"]
env.safe_run("%s update" % brew_cmd)
current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()])
for repo in formula_repos:
if repo not in current_taps:
env.safe_run("%s tap %s" % (brew_cmd, repo))
current_pkgs = set([x.strip() for x in env.safe_run_output("%s list" % brew_cmd).split()])
outdated_pkgs = set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()])
for pkg in packages:
if pkg in outdated_pkgs:
brew_subcmd = "upgrade"
elif pkg in current_pkgs:
brew_subcmd = None
else:
brew_subcmd = "install"
if brew_subcmd:
env.safe_run("%s %s %s" % (brew_cmd, brew_subcmd, pkg))
def install_packages(env, to_install=None, packages=None):
if shared._is_anaconda(env):
conda_bin = shared._conda_cmd(env)
config_file = get_config_file(env, "packages-conda.yaml")
if config_file.base is None and packages is None:
packages = []
else:
if to_install:
(packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
with open(config_file.base) as in_handle:
channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])])
conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals())))
if len(packages) > 0:
pkgs_str = " ".join(packages)
env.safe_run("{conda_bin} install -y {channels} {pkgs_str}".format(**locals()))
for package in packages:
_symlink_bin(package, env, conda_info, conda_bin)
for pkg in ["python", "conda", "pip"]:
_symlink_bin(pkg, env, conda_info, conda_bin, [pkg], "bcbio_")
# remove packages that can cause failures
# curl https://github.com/ContinuumIO/anaconda-issues/issues/72
problem_packages = ["curl"]
pkgs_str = " ".join(problem_packages)
with settings(warn_only=True):
env.safe_run("{conda_bin} uninstall -y {pkgs_str}".format(**locals()))
def install_packages(env, to_install=None, packages=None):
"""Install packages using the home brew package manager.
Handles upgrading brew, tapping required repositories and installing or upgrading
packages as appropriate.
`to_install` is a CloudBioLinux compatible set of top level items to add,
alternatively `packages` is a list of raw package names.
"""
config_file = get_config_file(env, "packages-homebrew.yaml")
if to_install:
(packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
# if we have no packages to install, do not try to install or update brew
if len(packages) == 0:
return
system.install_homebrew(env)
brew_cmd = _brew_cmd(env)
formula_repos = ["homebrew/science", "chapmanb/cbl"]
current_taps = set([x.strip() for x in env.safe_run_output("%s tap" % brew_cmd).split()])
_safe_update(env, brew_cmd, formula_repos, current_taps)
for repo in formula_repos:
if repo not in current_taps:
env.safe_run("%s tap %s" % (brew_cmd, repo))
env.safe_run("%s tap --repair" % brew_cmd)
ipkgs = {"outdated": set([x.strip() for x in env.safe_run_output("%s outdated" % brew_cmd).split()]),
"current": _get_current_pkgs(env, brew_cmd)}
_install_brew_baseline(env, brew_cmd, ipkgs, packages)
for pkg_str in packages:
_install_pkg(env, pkg_str, brew_cmd, ipkgs)
def _setup_env(env):
"""
Setup the system environment required to run CloudMan. This means
installing required system-level packages (as defined in CBL's
``packages.yaml``, or a flavor thereof) and Python dependencies
(i.e., libraries) as defined in CloudMan's ``requirements.txt`` file.
"""
# Get and install required system packages
if env.distribution in ["debian", "ubuntu"]:
config_file = get_config_file(env, "packages.yaml")
(packages, _) = _yaml_to_packages(config_file.base, 'cloudman')
# Allow flavors to modify the package list
packages = env.flavor.rewrite_config_items("packages", packages)
_setup_apt_automation()
_apt_packages(pkg_list=packages)
elif env.distribution in ["centos", "scientificlinux"]:
env.logger.warn("No CloudMan system package dependencies for CentOS")
pass
# Get and install required Python libraries
with _make_tmp_dir() as work_dir:
with cd(work_dir):
url = os.path.join(CM_REPO_ROOT_URL, 'requirements.txt')
_create_python_virtualenv(env, 'CM', reqs_url=url)
# Add a custom vimrc
vimrc_url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', 'vimrc')
remote_file = '/etc/vim/vimrc'
if env.safe_exists("/etc/vim"):
env.safe_sudo("wget --output-document=%s %s" % (remote_file, vimrc_url))
env.logger.debug("Added a custom vimrc to {0}".format(remote_file))
# Setup profile
aliases = ['alias lt="ls -ltr"', 'alias ll="ls -l"']
for alias in aliases:
_add_to_profiles(alias, ['/etc/bash.bashrc'])
env.logger.info("Done setting up CloudMan's environment")
def _setup_env(env):
""" Setup the system environment required to run CloudMan. This primarily
refers to installing required Python dependencies (ie, libraries) as
defined in CloudMan's requirements.txt file.
"""
# Get and install required system packages
if env.distribution in ["debian", "ubuntu"]:
conf_file = 'config.yaml'
url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', conf_file)
cf = urllib.urlretrieve(url)
(packages, _) = _yaml_to_packages(cf[0], 'cloudman')
_apt_packages(pkg_list=packages)
elif env.distibution in ["centos", "scientificlinux"]:
env.logger.warn("No CloudMan system package dependencies for CentOS")
pass
reqs_file = 'requirements.txt'
with _make_tmp_dir() as work_dir:
with cd(work_dir):
# Get and install requried Python libraries
url = os.path.join(CM_REPO_ROOT_URL, reqs_file)
run("wget --output-document=%s %s" % (reqs_file, url))
sudo("pip install --upgrade --requirement={0}".format(reqs_file))
# Add a custom vimrc
vimrc_url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', 'vimrc')
remote_file = '/etc/vim/vimrc'
sudo("wget --output-document=%s %s" % (remote_file, vimrc_url))
env.logger.debug("Added a custom vimrc to {0}".format(remote_file))
env.logger.debug("Done setting up CloudMan's environment")
def install_custom(p, automated=False, pkg_to_group=None):
"""Install a single custom package by name.
This method fetches names from custom.yaml that delegate to a method
in the custom/name.py program.
fab install_custom_package:package_name
"""
_setup_logging(env)
env.logger.info("Install custom software packages")
if not automated:
_parse_fabricrc()
_setup_edition(env)
_setup_distribution_environment()
_create_local_paths()
pkg_config = os.path.join(env.config_dir, "custom.yaml")
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
try:
env.logger.debug("Import %s" % p)
mod = __import__("cloudbio.custom.%s" % pkg_to_group[p],
fromlist=["cloudbio", "custom"])
except ImportError:
raise ImportError("Need to write a %s module in custom." %
pkg_to_group[p])
replace_chars = ["-"]
try:
for to_replace in replace_chars:
p = p.replace(to_replace, "_")
fn = getattr(mod, "install_%s" % p)
except AttributeError:
raise ImportError("Need to write a install_%s function in custom.%s"
% (p, pkg_to_group[p]))
fn(env)
def _custom_installs(to_install):
if not exists(env.local_install):
run("mkdir -p %s" % env.local_install)
pkg_config = os.path.join(env.config_dir, "custom.yaml")
packages, pkg_to_group = _yaml_to_packages(pkg_config, to_install)
for p in env.flavor.rewrite_config_items("custom", packages):
install_custom(p, True, pkg_to_group)
def _apt_packages(to_install):
"""Install packages available via apt-get.
"""
env.logger.info("Update and install all packages")
pkg_config_file = os.path.join(env.config_dir, "packages.yaml")
subs_pkg_config_file = os.path.join(env.config_dir, "packages-%s.yaml" %
env.distribution)
if not os.path.exists(subs_pkg_config_file): subs_pkg_config_file = None
sudo("apt-get update") # Always update
env.edition.apt_upgrade_system()
# Retrieve final package names
(packages, _) = _yaml_to_packages(pkg_config_file, to_install,
subs_pkg_config_file)
# At this point allow the Edition to rewrite the package list -
# this is shared within and between editions.
# Ref: https://github.com/chapmanb/cloudbiolinux/pull/10#issuecomment-1616423
packages = env.edition.rewrite_config_items("packages", packages)
# At this point allow the Flavor to rewrite the package list
packages = env.flavor.rewrite_config_items("packages", packages)
# A single line install is much faster - note that there is a max
# for the command line size, so we do 30 at a time
group_size = 30
i = 0
env.logger.info("Updating %i packages" % len(packages))
while i < len(packages):
sudo("apt-get -y --force-yes install %s" % " ".join(packages[i:i+group_size]))
i += group_size
sudo("apt-get clean")
def install_packages(env, to_install=None, packages=None):
if shared._is_anaconda(env):
conda_bin = shared._conda_cmd(env)
config_file = get_config_file(env, "packages-conda.yaml")
if config_file.base is None and packages is None:
packages = []
else:
if to_install:
(packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
with open(config_file.base) as in_handle:
channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])])
conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals())))
# Transition change -- ensure installed perl is perl-threaded
system_packages = ["perl", "perl-threaded"]
pkgs_str = " ".join(system_packages)
with settings(warn_only=True):
env.safe_run("{conda_bin} uninstall -y {pkgs_str}".format(**locals()))
# install our customized packages
if len(packages) > 0:
pkgs_str = " ".join(packages)
env.safe_run("{conda_bin} install -y {channels} {pkgs_str}".format(**locals()))
for package in packages:
_link_bin(package, env, conda_info, conda_bin)
for pkg in ["python", "conda", "pip"]:
_link_bin(pkg, env, conda_info, conda_bin, [pkg], "bcbio_")
# remove packages we want the system to supply
# curl https://github.com/ContinuumIO/anaconda-issues/issues/72
system_packages = ["curl"]
pkgs_str = " ".join(system_packages)
with settings(warn_only=True):
env.safe_run("{conda_bin} uninstall -y {pkgs_str}".format(**locals()))
def _custom_installs(to_install, ignore=None):
if not env.safe_exists(env.local_install) and env.local_install:
env.safe_run("mkdir -p %s" % env.local_install)
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, to_install)
packages = [p for p in packages if ignore is None or p not in ignore]
for p in env.flavor.rewrite_config_items("custom", packages):
install_custom(p, True, pkg_to_group)
def _apt_packages(to_install=None, pkg_list=None, pkg_config_file_path=None):
"""
Install packages available via apt-get.
Note that ``to_install`` and ``pkg_list`` arguments cannot be used simultaneously.
:type to_install: list
:param to_install: A list of strings (ie, groups) present in the ``main.yaml``
config file that will be used to filter out the specific
packages to be installed.
:type pkg_list: list
:param pkg_list: An explicit list of packages to install. No other files,
flavors, or editions are considered.
:type pkg_config_file_path: string
:param pkg_config_file_path: Allows a custom path to be specified where
``packages.yaml`` and ``packages-[dist].yaml``
are stored. Note that the file names cannot
be customized, only the path.
"""
env.logger.info("Update the system")
sudo("apt-get update") # Always update
if to_install is not None:
env.logger.info("Will install all packages (as listed in packages*.yaml "
"files in {0})".format(pkg_config_file_path))
if pkg_config_file_path is None:
pkg_config_file_path = env.config_dir
pkg_config_file = os.path.join(pkg_config_file_path, "packages.yaml")
subs_pkg_config_file = os.path.join(pkg_config_file_path, "packages-%s.yaml" %
env.distribution)
if not os.path.exists(subs_pkg_config_file): subs_pkg_config_file = None
env.edition.apt_upgrade_system()
# Retrieve final package names
(packages, _) = _yaml_to_packages(pkg_config_file, to_install,
subs_pkg_config_file)
# At this point allow the Edition to rewrite the package list -
# this is shared within and between editions.
# Ref: https://github.com/chapmanb/cloudbiolinux/pull/10#issuecomment-1616423
packages = env.edition.rewrite_config_items("packages", packages)
# At this point allow the Flavor to rewrite the package list
packages = env.flavor.rewrite_config_items("packages", packages)
elif pkg_list is not None:
env.logger.info("Will install specific packages: {0}".format(pkg_list))
packages = pkg_list
else:
raise ValueError("Need a file with packages or a list of packages")
# A single line install is much faster - note that there is a max
# for the command line size, so we do 30 at a time
group_size = 30
i = 0
env.logger.info("Installing %i packages" % len(packages))
while i < len(packages):
env.logger.info("Package install progress: {0}/{1}".format(i, len(packages)))
sudo("apt-get -y --force-yes install %s" % " ".join(packages[i:i+group_size]))
i += group_size
sudo("apt-get clean")
def install_packages(env):
config_file = get_config_file(env, "perl-libs.yaml")
(packages, _) = _yaml_to_packages(config_file.base, subs_yaml_file=config_file.dist, namesort=False)
cpanm_cmd = find_cmd(env, "cpanm", "--version")
for package in packages:
if package.count("==") > 1:
_install_from_url(env, cpanm_cmd, package)
else:
_install_from_cpan(env, cpanm_cmd, package)
def _provision_puppet_classes(to_install, ignore=None):
"""
Much like _custom_installs, read config file, determine what to install,
and install it.
"""
pkg_config = get_config_file(env, "puppet_classes.yaml").base
packages, _ = _yaml_to_packages(pkg_config, to_install)
packages = [p for p in packages if ignore is None or p not in ignore]
classes = [recipe for recipe in env.flavor.rewrite_config_items("puppet_classes", packages)]
def install_packages(env, to_install=None, packages=None):
if shared._is_anaconda(env):
conda_bin = shared._conda_cmd(env)
config_file = get_config_file(env, "packages-conda.yaml")
if to_install:
(packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
with open(config_file.base) as in_handle:
channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])])
if len(packages) > 0:
for pkg in packages:
env.safe_run("{conda_bin} install -y {channels} {pkg}".format(**locals()))
def _provision_chef_recipes(to_install, ignore=None):
"""
Much like _custom_installs, read config file, determine what to install,
and install it.
"""
pkg_config = get_config_file(env, "chef_recipes.yaml").base
packages, _ = _yaml_to_packages(pkg_config, to_install)
packages = [p for p in packages if ignore is None or p not in ignore]
recipes = [recipe for recipe in env.flavor.rewrite_config_items("chef_recipes", packages)]
if recipes: # Don't bother running chef if nothing to configure
install_chef_recipe(recipes, True)
def _nix_packages(to_install):
"""Install packages available via nixpkgs (optional)
"""
if env.nixpkgs:
env.logger.info("Update and install NixPkgs packages")
pkg_config_file = get_config_file(env, "packages-nix.yaml").base
sudo("nix-channel --update")
# Retrieve final package names
(packages, _) = _yaml_to_packages(pkg_config_file, to_install)
packages = env.flavor.rewrite_config_items("packages", packages)
for p in packages:
sudo("nix-env -b -i %s" % p)
def _custom_installs(to_install, ignore=None, add=None):
if not env.safe_exists(env.local_install) and env.local_install:
env.safe_run("mkdir -p %s" % env.local_install)
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, to_install)
packages = [p for p in packages if ignore is None or p not in ignore]
if add is not None:
for key, vals in add.iteritems():
for v in vals:
pkg_to_group[v] = key
packages.append(v)
for p in env.flavor.rewrite_config_items("custom", packages):
install_custom(p, True, pkg_to_group)
def _yum_packages(to_install):
"""Install rpm packages available via yum.
"""
pkg_config = os.path.join(env.config_dir, "packages-yum.yaml")
with settings(warn_only=True):
sudo("yum check-update")
sudo("yum -y upgrade")
# Retrieve packages to get and install each of them
(packages, _) = _yaml_to_packages(pkg_config, to_install)
# At this point allow the Flavor to rewrite the package list
packages = env.flavor.rewrite_config_items("packages", packages)
for package in packages:
sudo("yum -y install %s" % package)
def install_custom(p, automated=False, pkg_to_group=None, flavor=None):
"""
Install a single custom program or package by name.
This method fetches program name from ``config/custom.yaml`` and delegates
to a method in ``custom/*name*.py`` to proceed with the installation.
Alternatively, if a program install method is defined in the appropriate
package, it will be called directly (see param ``p``).
Usage: fab [-i key] [-u user] -H host install_custom:program_name
:type p: string
:param p: A name of the custom program to install. This has to be either a name
that is listed in ``custom.yaml`` as a subordinate to a group name or a
program name whose install method is defined in either ``cloudbio`` or
``custom`` packages
(e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``).
:type automated: bool
:param automated: If set to True, the environment is not loaded and reading of
the ``custom.yaml`` is skipped.
"""
_setup_logging(env)
p = p.lower() # All packages listed in custom.yaml are in lower case
time_start = _print_time_stats("Custom install for '{0}'".format(p), "start")
if not automated:
_configure_fabric_environment(env, flavor)
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
try:
env.logger.debug("Import %s" % p)
# Allow direct calling of a program install method, even if the program
# is not listed in the custom list (ie, not contained as a key value in
# pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman.
mod_name = pkg_to_group[p] if p in pkg_to_group else p
mod = __import__("cloudbio.custom.%s" % mod_name,
fromlist=["cloudbio", "custom"])
except ImportError:
raise ImportError("Need to write a %s module in custom." %
pkg_to_group[p])
replace_chars = ["-"]
try:
for to_replace in replace_chars:
p = p.replace(to_replace, "_")
fn = getattr(mod, "install_%s" % p)
except AttributeError:
raise ImportError("Need to write a install_%s function in custom.%s"
% (p, pkg_to_group[p]))
fn(env)
_print_time_stats("Custom install for '%s'" % p, "end", time_start)
def install_custom(p, automated=False, pkg_to_group=None):
"""Install a single custom package by name.
This method fetches names from custom.yaml that delegate to a method
in the custom/name.py program. Alternatively, if a program install method is
defined in approapriate package, it will be called directly (see param p).
Usage: fab [-i key] [-u user] -H host install_custom:program_name
:type p: string
:param p: A name of a custom program to install. This has to be either a name
that is listed in custom.yaml as a subordinate to a group name or a
program name whose install method is defined in either cloudbio or
custom packages (eg, install_cloudman).
:type automated: bool
:param automated: If set to True, the environment is not loaded and reading of
the custom.yaml is skipped.
"""
_setup_logging(env)
p = p.lower() # All packages are listed in custom.yaml are in lower case
time_start = _print_time_stats("Custom install for '{0}'".format(p), "start")
if not automated:
_parse_fabricrc()
_setup_edition(env)
_setup_distribution_environment()
_create_local_paths()
pkg_config = os.path.join(env.config_dir, "custom.yaml")
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
try:
env.logger.debug("Import %s" % p)
# Allow direct calling of a program install method, even if the program
# is not listed in the custom list (ie, not contained as a key value in
# pkg_to_group). For an example, see 'install_cloudman' or use p=cloudman.
mod_name = pkg_to_group[p] if p in pkg_to_group else p
mod = __import__("cloudbio.custom.%s" % mod_name,
fromlist=["cloudbio", "custom"])
except ImportError:
raise ImportError("Need to write a %s module in custom." %
pkg_to_group[p])
replace_chars = ["-"]
try:
for to_replace in replace_chars:
p = p.replace(to_replace, "_")
fn = getattr(mod, "install_%s" % p)
except AttributeError:
raise ImportError("Need to write a install_%s function in custom.%s"
% (p, pkg_to_group[p]))
fn(env)
_print_time_stats("Custom install for '%s'" % p, "end", time_start)
def _yum_packages(to_install):
"""Install rpm packages available via yum.
"""
if env.distribution == "scientificlinux":
package_file = "packages-scientificlinux.yaml"
else:
package_file = "packages-yum.yaml"
pkg_config = get_config_file(env, package_file).base
with settings(warn_only=True):
sudo("yum check-update")
sudo("yum -y upgrade")
# Retrieve packages to get and install each of them
(packages, _) = _yaml_to_packages(pkg_config, to_install)
# At this point allow the Flavor to rewrite the package list
packages = env.flavor.rewrite_config_items("packages", packages)
for package in packages:
sudo("yum -y install %s" % package)
def install_packages(env, to_install=None, packages=None):
if shared._is_anaconda(env):
conda_bin = shared._conda_cmd(env)
if hasattr(env, "conda_yaml"):
Config = collections.namedtuple("Config", "base dist")
config_file = Config(base=env.conda_yaml, dist=None)
else:
config_file = get_config_file(env, "packages-conda.yaml")
if config_file.base is None and packages is None:
packages = []
else:
if to_install:
(packages, _) = _yaml_to_packages(config_file.base, to_install, config_file.dist)
with open(config_file.base) as in_handle:
channels = " ".join(["-c %s" % x for x in yaml.safe_load(in_handle).get("channels", [])])
conda_envs = _create_environments(env, conda_bin, packages)
conda_info = json.loads(env.safe_run_output("{conda_bin} info --json".format(**locals())))
# Uninstall old R packages that clash with updated versions
# Temporary fix to allow upgrades from older versions that have migrated
# r-tximport is now bioconductor-tximport
# py2cairo is incompatible with r 3.4.1
# libedit pins to curses 6.0 but bioconda requires 5.9
for problem in ["r-tximport", "py2cairo", "libedit"]:
cur_packages = [x["name"] for x in
json.loads(env.safe_run_output("{conda_bin} list --json {problem}".format(**locals())))]
if problem in cur_packages:
env.safe_run("{conda_bin} remove --force -y {problem}".format(**locals()))
# install our customized packages
if len(packages) > 0:
for env_name, env_packages in _split_by_condaenv(packages):
if env_name:
assert env_name in conda_envs, (env_name, conda_envs)
env_str = "-n %s" % env_name
else:
env_str = ""
pkgs_str = " ".join(env_packages)
env.safe_run("{conda_bin} install --quiet -y {env_str} {channels} {pkgs_str}".format(**locals()))
conda_pkg_list = json.loads(env.safe_run_output(
"{conda_bin} list --json {env_str}".format(**locals())))
for package in env_packages:
_link_bin(package, env, conda_info, conda_bin, conda_pkg_list,
conda_envdir=conda_envs.get(env_name))
conda_pkg_list = json.loads(env.safe_run_output("{conda_bin} list --json".format(**locals())))
for pkg in ["python", "conda", "pip"]:
_link_bin(pkg, env, conda_info, conda_bin, conda_pkg_list, files=[pkg], prefix="bcbio_")
def _apt_packages(to_install=None, pkg_list=None):
"""
Install packages available via apt-get.
Note that ``to_install`` and ``pkg_list`` arguments cannot be used simultaneously.
:type to_install: list
:param to_install: A list of strings (ie, groups) present in the ``main.yaml``
config file that will be used to filter out the specific
packages to be installed.
:type pkg_list: list
:param pkg_list: An explicit list of packages to install. No other files,
flavors, or editions are considered.
"""
if env.edition.short_name not in ["minimal"]:
env.logger.info("Update the system")
with settings(warn_only=True):
sudo("apt-get update")
if to_install is not None:
config_file = get_config_file(env, "packages.yaml")
env.edition.apt_upgrade_system()
(packages, _) = _yaml_to_packages(config_file.base, to_install,
config_file.dist)
# Allow editions and flavors to modify the package list
packages = env.edition.rewrite_config_items("packages", packages)
packages = env.flavor.rewrite_config_items("packages", packages)
elif pkg_list is not None:
env.logger.info("Will install specific packages: {0}".format(pkg_list))
packages = pkg_list
else:
raise ValueError("Need a file with packages or a list of packages")
# A single line install is much faster - note that there is a max
# for the command line size, so we do 30 at a time
group_size = 30
i = 0
env.logger.info("Installing %i packages" % len(packages))
while i < len(packages):
env.logger.info("Package install progress: {0}/{1}".format(
i, len(packages)))
sudo("apt-get -y --force-yes install %s" % " ".join(
packages[i:i + group_size]))
i += group_size
sudo("apt-get clean")
def install_custom(p, automated=False, pkg_to_group=None, flavor=None):
"""
Install a single custom program or package by name.
This method fetches program name from ``config/custom.yaml`` and delegates
to a method in ``custom/*name*.py`` to proceed with the installation.
Alternatively, if a program install method is defined in the appropriate
package, it will be called directly (see param ``p``).
Usage: fab [-i key] [-u user] -H host install_custom:program_name
:type p: string
:param p: A name of the custom program to install. This has to be either a name
that is listed in ``custom.yaml`` as a subordinate to a group name or a
program name whose install method is defined in either ``cloudbio`` or
``custom`` packages
(e.g., ``cloudbio/custom/cloudman.py -> install_cloudman``).
:type automated: bool
:param automated: If set to True, the environment is not loaded and reading of
the ``custom.yaml`` is skipped.
"""
p = p.lower() # All packages listed in custom.yaml are in lower case
if not automated:
_setup_logging(env)
_configure_fabric_environment(env, flavor, ignore_distcheck=True)
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
time_start = _print_time_stats("Custom install for '{0}'".format(p), "start")
fn = _custom_install_function(env, p, pkg_to_group)
fn(env)
## TODO: Replace the previous 4 lines with the following one, barring
## objections. Slightly different behavior because pkg_to_group will be
## loaded regardless of automated if it is None, but IMO this shouldn't
## matter because the following steps look like they would fail if
## automated is True and pkg_to_group is None.
# _install_custom(p, pkg_to_group)
_print_time_stats("Custom install for '%s'" % p, "end", time_start)
def install_in(conda_bin, system_installdir, config_file=None, packages=None):
"""Install packages inside a given anaconda directory.
New approach, local only and not dependent on fabric.
"""
if config_file is None and packages is None:
packages = []
check_channels = []
else:
(packages, _) = _yaml_to_packages(config_file)
with open(config_file) as in_handle:
check_channels = yaml.safe_load(in_handle).get("channels", [])
channels = " ".join(["-c %s" % x for x in check_channels])
conda_envs = _create_environments(conda_bin, packages)
for env_dir in conda_envs.values():
_clean_environment(env_dir)
conda_info = json.loads(subprocess.check_output("{conda_bin} info --json".format(**locals()), shell=True))
# Uninstall old R packages that clash with updated versions
# Temporary fix to allow upgrades from older versions that have migrated
# r-tximport is now bioconductor-tximport
# py2cairo is incompatible with r 3.4.1+
problems = ["r-tximport", "py2cairo"]
# Add packages migrated into separate environments, like python2
for env_name, env_packages in _split_by_condaenv(packages):
if env_name:
problems += env_packages
if problems:
print("Checking for problematic or migrated packages in default environment")
cur_packages = [x["name"] for x in
json.loads(subprocess.check_output("%s list --json" % (conda_bin), shell=True))
if x["name"] in problems and x["channel"] in check_channels]
if cur_packages:
print("Found packages that moved from default environment: %s" % ", ".join(cur_packages))
problems = " ".join(cur_packages)
subprocess.check_call("{conda_bin} remove -y {problems}".format(**locals()), shell=True)
# install our customized packages
if len(packages) > 0:
for env_name, env_packages in _split_by_condaenv(packages):
print("# Installing into conda environment %s: %s" % (env_name or "default", ", ".join(env_packages)))
if env_name:
assert env_name in conda_envs, (env_name, conda_envs)
env_str = "-n %s" % env_name
else:
env_str = ""
pkgs_str = " ".join(["'%s'" % x for x in sorted(env_packages)])
py_version = ENV_PY_VERSIONS[env_name]
# During openssl transition, pin to 1.1.1 to avoid slow resolve times
extra_pins = "'openssl=1.1.1b'"
if "deepvariant" in env_packages:
# Ignore /etc/boto.cfg which creates conflicts with conda gsutils
# https://github.com/GoogleCloudPlatform/gsutil/issues/516
exports = "export BOTO_CONFIG=/ignoreglobal && "
else:
exports = ""
subprocess.check_call("{exports}{conda_bin} install -y {env_str} {channels} "
"{py_version} {extra_pins} {pkgs_str}".format(**locals()), shell=True)
conda_pkg_list = json.loads(subprocess.check_output(
"{conda_bin} list --json {env_str}".format(**locals()), shell=True))
for package in env_packages:
_link_bin(package, system_installdir, conda_info, conda_bin, conda_pkg_list,
conda_envdir=conda_envs.get(env_name))
conda_pkg_list = json.loads(subprocess.check_output("{conda_bin} list --json".format(**locals()), shell=True))
for pkg in ["python", "conda", "pip"]:
_link_bin(pkg, system_installdir, conda_info, conda_bin, conda_pkg_list, files=[pkg], prefix="bcbio_")
def _install_custom(p, pkg_to_group=None):
if pkg_to_group is None:
pkg_config = get_config_file(env, "custom.yaml").base
packages, pkg_to_group = _yaml_to_packages(pkg_config, None)
fn = _custom_install_function(env, p, pkg_to_group)
fn(env)
