def _link_bin(package, env, conda_info, conda_bin, files=None, prefix=""):
"""Link files installed in the bin directory into the install directory.
This is imperfect but we're trying not to require injecting everything in the anaconda
directory into a user's path.
"""
package = package.split("=")[0]
final_bindir = os.path.join(env.system_install, "bin")
base_bindir = os.path.dirname(conda_bin)
# resolve any symlinks in the final and base heirarchies
with quiet():
final_bindir = env.safe_run_output("cd %s && pwd -P" % final_bindir)
base_bindir = env.safe_run_output("cd %s && pwd -P" % base_bindir)
for pkg_subdir in json.loads(
env.safe_run_output(
"{conda_bin} list --json -f {package}".format(**locals()))):
pkg_subdir = pkg_subdir.split("::")[-1]
for pkg_dir in conda_info["pkgs_dirs"]:
pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin")
if env.safe_exists(pkg_bindir):
if not files:
with quiet():
files = env.safe_run_output(
"ls -1 {pkg_bindir}".format(**locals())).split()
for fname in files:
# symlink to the original file in the /anaconda/bin directory
# this could be a hard or soft link
base_fname = os.path.join(base_bindir, fname)
if os.path.exists(base_fname) and os.path.lexists(
base_fname):
_do_link(
base_fname,
os.path.join(final_bindir,
"%s%s" % (prefix, fname)))
def _link_bin(package, env, conda_info, conda_bin, files=None, prefix=""):
"""Link files installed in the bin directory into the install directory.
This is imperfect but we're trying not to require injecting everything in the anaconda
directory into a user's path.
"""
package = package.split("=")[0]
final_bindir = os.path.join(env.system_install, "bin")
base_bindir = os.path.dirname(conda_bin)
# resolve any symlinks in the final and base heirarchies
with quiet():
final_bindir = env.safe_run_output("cd %s && pwd -P" % final_bindir)
base_bindir = env.safe_run_output("cd %s && pwd -P" % base_bindir)
for pkg_subdir in json.loads(env.safe_run_output("{conda_bin} list --json -f {package}".format(**locals()))):
for pkg_dir in conda_info["pkgs_dirs"]:
pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin")
if env.safe_exists(pkg_bindir):
if not files:
with quiet():
files = env.safe_run_output("ls -1 {pkg_bindir}".format(**locals())).split()
for fname in files:
# symlink to the original file in the /anaconda/bin directory
# this could be a hard or soft link
base_fname = os.path.join(base_bindir, fname)
if os.path.exists(base_fname) and os.path.lexists(base_fname):
_do_link(base_fname,
os.path.join(final_bindir, "%s%s" % (prefix, fname)))
def _is_anaconda(env):
"""Check if we have a conda command or are in an anaconda subdirectory.
"""
with quiet():
conda = _conda_cmd(env)
has_conda = conda and env.safe_run_output("%s -h" % conda).startswith("usage: conda")
with quiet():
full_pip = env.safe_run_output("which %s" % _pip_cmd(env))
in_anaconda_dir = "/anaconda/" in full_pip
return has_conda or in_anaconda_dir
def _is_anaconda(env):
"""Check if we have a conda command or are in an anaconda subdirectory.
"""
with quiet():
conda = _conda_cmd(env)
has_conda = conda and env.safe_run_output(
"%s -h" % conda).startswith("usage: conda")
with quiet():
full_pip = env.safe_run_output("which %s" % _pip_cmd(env))
in_anaconda_dir = "/anaconda/" in full_pip
return has_conda or in_anaconda_dir
def _is_anaconda(env):
"""Check if we have a conda command or are in an anaconda subdirectory.
"""
with quiet():
conda = _conda_cmd(env)
has_conda = conda and env.safe_run_output("%s -h" % conda).startswith("usage: conda")
with quiet():
try:
full_pip = env.safe_run_output("which %s" % _pip_cmd(env))
except ValueError:
full_pip = None
in_anaconda_dir = full_pip and full_pip.succeeded and "/anaconda/" in full_pip
return has_conda or in_anaconda_dir
def _install_brew_baseline(env, brew_cmd, ipkgs, packages):
"""Install baseline brew components not handled by dependency system.
- Installation of required Perl libraries.
- Ensures installed samtools does not overlap with bcftools
- Upgrades any package dependencies
"""
for dep in ["openssl"]:
_safe_link_pkg(env, dep, brew_cmd)
for dep in ["expat", "pkg-config", "xz"]:
_install_pkg(env, dep, brew_cmd, ipkgs)
# check if we have an older git and need to install it from brew
git_version = None
with quiet():
with settings(warn_only=True):
git_version = env.safe_run_output("git --version").strip().split()[-1]
if git_version and LooseVersion(git_version) < LooseVersion("1.7"):
_install_pkg(env, "git", brew_cmd, ipkgs)
for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba
env.safe_run("{brew_cmd} remove --force {dep}".format(**locals()))
# if installing samtools, avoid bcftools conflicts
if len([x for x in packages if x.find("samtools") >= 0]):
with settings(warn_only=True):
def _has_prog(prog):
try:
return int(env.safe_run_output("{brew_cmd} list samtools | grep -c {prog} | cat".format(
brew_cmd=brew_cmd, prog=prog)))
except ValueError:
return 0
if any(_has_prog(p) for p in ["bctools", "vcfutils.pl"]):
env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools"))
ipkgs["current"].pop("samtools", None)
_install_pkg_latest(env, "samtools", ["--without-curses"], brew_cmd, ipkgs)
for dependency in ["htslib"]:
if dependency in packages:
if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"]
or dependency not in ipkgs["current"]):
_install_pkg_latest(env, dependency, [], brew_cmd, ipkgs)
if "cpanminus" in packages:
_install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs)
_install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs)
cpan.install_packages(env)
# Ensure paths we may have missed on install are accessible to regular user
if env.use_sudo:
paths = ["share", "share/java"]
for path in paths:
with quiet():
test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path,
env.system_install, path))
if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _install_brew_baseline(env, brew_cmd, ipkgs, packages):
"""Install baseline brew components not handled by dependency system.
- Installation of required Perl libraries.
- Ensures installed samtools does not overlap with bcftools
- Upgrades any package dependencies
"""
for dep in ["openssl"]:
_safe_link_pkg(env, dep, brew_cmd)
for dep in ["expat", "pkg-config", "cmake", "xz"]:
_install_pkg(env, dep, brew_cmd, ipkgs)
# check if we have an older git and need to install it from brew
git_version = None
with quiet():
with settings(warn_only=True):
git_version = env.safe_run_output("git --version").strip().split()[-1]
if git_version and LooseVersion(git_version) < LooseVersion("1.7"):
_install_pkg(env, "git", brew_cmd, ipkgs)
for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba
env.safe_run("{brew_cmd} remove --force {dep}".format(**locals()))
# if installing samtools, avoid bcftools conflicts
if len([x for x in packages if x.find("samtools") >= 0]):
with settings(warn_only=True):
def _has_prog(prog):
try:
return int(env.safe_run_output("{brew_cmd} list samtools | grep -c {prog} | cat".format(
brew_cmd=brew_cmd, prog=prog)))
except ValueError:
return 0
if any(_has_prog(p) for p in ["bctools", "vcfutils.pl"]):
env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools"))
ipkgs["current"].pop("samtools", None)
_install_pkg_latest(env, "samtools", ["--without-curses"], brew_cmd, ipkgs)
for dependency in ["htslib"]:
if dependency in packages:
if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"]
or dependency not in ipkgs["current"]):
_install_pkg_latest(env, dependency, [], brew_cmd, ipkgs)
if "cpanminus" in packages:
_install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs)
_install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs)
cpan.install_packages(env)
# Ensure paths we may have missed on install are accessible to regular user
if env.use_sudo:
paths = ["share", "share/java"]
for path in paths:
with quiet():
test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path,
env.system_install, path))
if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def install_homebrew(env):
"""Homebrew package manager for OSX and Linuxbrew for linux systems.
https://github.com/mxcl/homebrew
https://github.com/Homebrew/linuxbrew
"""
if env.distribution == "macosx":
with quiet():
test_brewcmd = env.safe_run("brew --version")
if not test_brewcmd.succeeded:
env.safe_run('ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/go/install)"')
else:
brewcmd = os.path.join(env.system_install, "bin", "brew")
with quiet():
test_brewcmd = env.safe_run("%s --version" % brewcmd)
if not test_brewcmd.succeeded or _linuxbrew_origin_problem(brewcmd):
with shared._make_tmp_dir() as tmp_dir:
with cd(tmp_dir):
if env.safe_exists("linuxbrew"):
env.safe_run("rm -rf linuxbrew")
for cleandir in ["Library", ".git"]:
if env.safe_exists("%s/%s" % (env.system_install, cleandir)):
env.safe_run("rm -rf %s/%s" % (env.system_install, cleandir))
env.safe_run("git clone https://github.com/Homebrew/linuxbrew.git")
with cd("linuxbrew"):
if not env.safe_exists(env.system_install):
env.safe_sudo("mkdir -p %s" % env.system_install)
env.safe_sudo("chown %s %s" % (env.user, env.system_install))
paths = ["bin", "etc", "include", "lib", "lib/pkgconfig", "Library",
"sbin", "share", "var", "var/log", "share/java", "share/locale",
"share/man", "share/man/man1", "share/man/man2",
"share/man/man3", "share/man/man4", "share/man/man5",
"share/man/man6", "share/man/man7", "share/man/man8",
"share/info", "share/doc", "share/aclocal",
"lib/python2.7/site-packages", "lib/python2.6/site-packages",
"lib/python3.2/site-packages", "lib/python3.3/site-packages",
"lib/perl5", "lib/perl5/site_perl"]
if not env.safe_exists("%s/bin" % env.system_install):
env.safe_sudo("mkdir -p %s/bin" % env.system_install)
for path in paths:
if env.safe_exists("%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
if not env.safe_exists("%s/Library" % env.system_install):
env.safe_run("mv Library %s" % env.system_install)
if not env.safe_exists("%s/.git" % env.system_install):
env.safe_run("mv .git %s" % env.system_install)
man_dir = "share/man/man1"
if not env.safe_exists("%s/%s" % (env.system_install, man_dir)):
env.safe_run("mkdir -p %s/%s" % (env.system_install, man_dir))
env.safe_run("mv -f %s/brew.1 %s/%s" % (man_dir, env.system_install, man_dir))
env.safe_run("mv -f bin/brew %s/bin" % env.system_install)
def _is_anaconda(env):
"""Check if we have a conda command or are in an anaconda subdirectory.
"""
with quiet():
conda = _conda_cmd(env)
has_conda = conda and env.safe_run_output(
"%s -h" % conda).startswith("usage: conda")
with quiet():
try:
full_pip = env.safe_run_output("which %s" % _pip_cmd(env))
except ValueError:
full_pip = None
in_anaconda_dir = full_pip and full_pip.succeeded and "/anaconda/" in full_pip
return has_conda or in_anaconda_dir
def _setup_deb_general():
"""Shared settings for different debian based/derived distributions.
"""
env.logger.debug("Debian-shared setup")
env.sources_file = "/etc/apt/sources.list.d/cloudbiolinux.list"
env.global_sources_file = "/etc/apt/sources.list"
env.apt_preferences_file = "/etc/apt/preferences"
if not hasattr(env, "python_version_ext"):
env.python_version_ext = ""
if not hasattr(env, "ruby_version_ext"):
env.ruby_version_ext = "1.9.1"
if not env.has_key("java_home"):
# Try to determine java location from update-alternatives
java_home = "/usr/lib/jvm/java-7-openjdk-amd64"
with quiet():
java_info = env.safe_run_output(
"update-alternatives --display java")
for line in java_info.split("\n"):
if line.strip().startswith("link currently points to"):
java_home = line.split()[-1].strip()
java_home = java_home.replace("/jre/bin/java", "")
env.java_home = java_home
shared_sources = [
"deb http://nebc.nerc.ac.uk/bio-linux/ unstable bio-linux", # Bio-Linux
"deb http://download.virtualbox.org/virtualbox/debian %s contrib", # virtualbox
]
return shared_sources
def _determine_distribution(env):
"""
Attempt to automatically determine the distribution of the target machine.
Currently works for Ubuntu, CentOS, Debian, Scientific Linux and Mac OS X.
"""
with quiet():
output = env.safe_run_output("cat /etc/*release").lower()
if output.find("distrib_id=ubuntu") >= 0:
return "ubuntu"
elif output.find("centos release") >= 0:
return "centos"
elif output.find("red hat enterprise linux server release") >= 0:
return "centos"
elif output.find("scientific linux release") >= 0:
return "scientificlinux"
elif env.safe_exists("/etc/debian_version"):
return "debian"
# check for file used by Python's platform.mac_ver
elif env.safe_exists("/System/Library/CoreServices/SystemVersion.plist"):
return "macosx"
else:
raise Exception(
"Attempt to automatically determine Linux distribution of target machine failed, please manually specify distribution in fabricrc.txt"
)
def _install_bottle(env, brew_cmd, pkg, ipkgs):
"""Install Linux bottles for brew packages that can be tricky to build.
"""
if env.distribution == "macosx": # Only Linux bottles, build away on Mac
return False
pkg_version, is_linked = _latest_pkg_version(env, brew_cmd, pkg)
install_version = ipkgs["current"].get(pkg)
if pkg_version == install_version: # Up to date
if not is_linked:
env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg))
return True
elif install_version or pkg in ipkgs["outdated"]:
env.safe_run("{brew_cmd} remove --force {pkg}".format(**locals()))
url = BOTTLE_URL.format(pkg=pkg, version=pkg_version)
brew_cachedir = env.safe_run_output("%s --cache" % brew_cmd)
brew_cellar = os.path.join(env.safe_run_output("%s --prefix" % brew_cmd), "Cellar")
with quiet():
env.safe_run("mkdir -p %s" % brew_cellar)
out_file = os.path.join(brew_cachedir, os.path.basename(url))
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
bottle_file = shared._remote_fetch(env, url, out_file=out_file,
allow_fail=True, samedir=True)
if bottle_file:
with cd(brew_cellar):
env.safe_run("tar -xf %s" % bottle_file)
env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg))
return True
else:
return False
def _determine_distribution(env):
"""
Attempt to automatically determine the distribution of the target machine.
Currently works for Ubuntu, CentOS, Debian, Scientific Linux and Mac OS X.
"""
with quiet():
output = env.safe_run_output("cat /etc/*release").lower()
if output.find("distrib_id=ubuntu") >= 0:
return "ubuntu"
elif output.find("centos release") >= 0:
return "centos"
elif output.find("centos linux release") >= 0:
return "centos"
elif output.find("red hat") >= 0:
return "centos"
elif output.find("fedora release") >= 0:
return "centos"
elif output.find("amzn") >= 0: # Amazon AMIs are Red-Hat based
return "centos"
elif output.find("suse linux") >= 0:
return "suse"
elif output.find("opensuse") >= 0:
return "suse"
elif output.find("scientific linux") >= 0:
return "scientificlinux"
elif env.safe_exists("/etc/debian_version"):
return "debian"
elif output.find("id=arch") >= 0:
return "arch"
# check for file used by Python's platform.mac_ver
elif env.safe_exists("/System/Library/CoreServices/SystemVersion.plist"):
return "macosx"
else:
raise Exception("Attempt to automatically determine Linux distribution of target machine failed, please manually specify distribution in fabricrc.txt")
def get_installed_version(env,
cmd,
version,
args=None,
stdout_flag=None,
stdout_index=-1):
"""Check if the given command is up to date with the provided version.
"""
if shared._executable_not_on_path(cmd):
return False
if args:
cmd = cmd + " " + " ".join(args)
with quiet():
path_safe = (
"export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:{s}/lib/pkgconfig && "
"export PATH=$PATH:{s}/bin && "
"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{s}/lib && ".format(
s=env.system_install))
out = env.safe_run_output(path_safe + cmd)
if stdout_flag:
iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index)
else:
iversion = out.strip()
iversion = _clean_version(iversion)
if " not found in the pkg-config search path" in iversion:
return False
return iversion
def _git_stash(env, brew_cmd):
"""Perform a safe git stash around an update.
This circumvents brews internal stash approach which doesn't work on older versions
of git and is sensitive to missing config.emails.
"""
brew_prefix = env.safe_run_output("{brew_cmd} --prefix".format(**locals()))
with cd(brew_prefix):
with quiet():
with settings(warn_only=True):
env.safe_run("git config user.email '*****@*****.**'")
check_diff = env.safe_run("git diff --quiet")
git_version = env.safe_run_output("git --version").strip().split()[-1]
if git_version and LooseVersion(git_version) < LooseVersion("1.7"):
if check_diff.return_code > 0:
with cd(brew_prefix):
with settings(warn_only=True):
env.safe_run("git stash --quiet")
try:
yield None
finally:
if check_diff.return_code > 0:
with cd(brew_prefix):
with settings(warn_only=True):
env.safe_run("git stash pop --quiet")
else:
yield None
def _safe_link_pkg(env, pkg_str, brew_cmd):
"""Link packages required for builds, but not necessarily installed
"""
with settings(warn_only=True):
with quiet():
env.safe_run(
"{brew_cmd} link --overwrite {pkg_str}".format(**locals()))
def _install_brew_baseline(env, brew_cmd, ipkgs, packages):
"""Install baseline brew components not handled by dependency system.
Handles installation of required Perl libraries.
"""
for dep in ["cpanminus", "expat"]:
_install_pkg_latest(env, dep, brew_cmd, ipkgs)
# if installing samtools, avoid conflicts with cbl and homebrew-science versions
if len([x for x in packages if x.find("samtools") >= 0]):
with settings(warn_only=True):
try:
has_bcftools = int(env.safe_run_output("{brew_cmd} list samtools | grep -c bcftools".format(
brew_cmd=brew_cmd)))
except:
has_bcftools = int(env.safe_run_output("{brew_cmd} list samtools | grep -c bcftools".format(
brew_cmd=brew_cmd)).split("\n")[-1].rstrip())
if has_bcftools:
env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools"))
cpanm_cmd = os.path.join(os.path.dirname(brew_cmd), "cpanm")
for perl_lib in ["Statistics::Descriptive"]:
env.safe_run("%s -i --notest --local-lib=%s '%s'" % (cpanm_cmd, env.system_install, perl_lib))
# Ensure paths we may have missed on install are accessible to regular user
if env.use_sudo:
paths = ["share", "share/java"]
for path in paths:
with quiet():
test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path,
env.system_install, path))
if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _install_bottle(env, brew_cmd, pkg, ipkgs):
"""Install Linux bottles for brew packages that can be tricky to build.
"""
if env.distribution == "macosx": # Only Linux bottles, build away on Mac
return False
pkg_version, is_linked = _latest_pkg_version(env, brew_cmd, pkg)
install_version = ipkgs["current"].get(pkg)
if pkg_version == install_version: # Up to date
if not is_linked:
env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg))
return True
elif install_version or pkg in ipkgs["outdated"]:
env.safe_run("{brew_cmd} remove --force {pkg}".format(**locals()))
url = BOTTLE_URL.format(pkg=pkg, version=pkg_version)
brew_cachedir = env.safe_run_output("%s --cache" % brew_cmd)
brew_cellar = os.path.join(env.safe_run_output("%s --prefix" % brew_cmd),
"Cellar")
with quiet():
env.safe_run("mkdir -p %s" % brew_cellar)
out_file = os.path.join(brew_cachedir, os.path.basename(url))
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
bottle_file = shared._remote_fetch(env,
url,
out_file=out_file,
allow_fail=True,
samedir=True)
if bottle_file:
with cd(brew_cellar):
env.safe_run("tar -xf %s" % bottle_file)
env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg))
return True
else:
return False
def _setup_deb_general():
"""Shared settings for different debian based/derived distributions.
"""
env.logger.debug("Debian-shared setup")
env.sources_file = "/etc/apt/sources.list.d/cloudbiolinux.list"
env.global_sources_file = "/etc/apt/sources.list"
env.apt_preferences_file = "/etc/apt/preferences"
if not hasattr(env, "python_version_ext"):
env.python_version_ext = ""
if not hasattr(env, "ruby_version_ext"):
env.ruby_version_ext = "1.9.1"
if not env.has_key("java_home"):
# Try to determine java location from update-alternatives
java_home = "/usr/lib/jvm/java-7-openjdk-amd64"
with quiet():
java_info = env.safe_run_output("update-alternatives --display java")
for line in java_info.split("\n"):
if line.strip().startswith("link currently points to"):
java_home = line.split()[-1].strip()
java_home = java_home.replace("/jre/bin/java", "")
env.java_home = java_home
shared_sources = [
"deb http://download.virtualbox.org/virtualbox/debian %s contrib", # virtualbox
]
return shared_sources
def _git_stash(env, brew_cmd):
"""Perform a safe git stash around an update.
This circumvents brews internal stash approach which doesn't work on older versions
of git and is sensitive to missing config.emails.
"""
brew_prefix = env.safe_run_output("{brew_cmd} --prefix".format(**locals()))
with cd(brew_prefix):
with quiet():
with settings(warn_only=True):
env.safe_run("git config user.email '*****@*****.**'")
check_diff = env.safe_run("git diff --quiet")
git_version = env.safe_run_output(
"git --version").strip().split()[-1]
if git_version and LooseVersion(git_version) < LooseVersion("1.7"):
if check_diff.return_code > 0:
with cd(brew_prefix):
with settings(warn_only=True):
env.safe_run("git stash --quiet")
try:
yield None
finally:
if check_diff.return_code > 0:
with cd(brew_prefix):
with settings(warn_only=True):
env.safe_run("git stash pop --quiet")
else:
yield None
def _install_brew_baseline(env, brew_cmd, ipkgs, packages):
"""Install baseline brew components not handled by dependency system.
- Installation of required Perl libraries.
- Ensures installed samtools does not overlap with bcftools
- Upgrades any package dependencies
"""
for dep in ["expat"]:
_install_pkg_latest(env, dep, [], brew_cmd, ipkgs)
# if installing samtools, avoid bcftools conflicts
if len([x for x in packages if x.find("samtools") >= 0]):
with settings(warn_only=True):
def _has_prog(prog):
try:
return int(
env.safe_run_output(
"{brew_cmd} list samtools | grep -c {prog}".format(brew_cmd=brew_cmd, prog=prog)
)
)
except ValueError:
return 0
if any(_has_prog(p) for p in ["bctools", "vcfutils.pl"]):
env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools"))
ipkgs["current"].pop("samtools", None)
_install_pkg_latest(env, "samtools", ["--devel"], brew_cmd, ipkgs)
for dependency in ["htslib", "libmaus", "cmake"]:
if dependency in packages:
if (
dependency in ipkgs["outdated"]
or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"]
or dependency not in ipkgs["current"]
):
_install_pkg_latest(env, dependency, [], brew_cmd, ipkgs)
if "cpanminus" in packages:
_install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs)
cpanm_cmd = os.path.join(os.path.dirname(brew_cmd), "cpanm")
for perl_lib in [
"Statistics::Descriptive",
"Archive::Extract",
"Archive::Zip",
"Archive::Tar",
"DBI",
"LWP::Simple",
"LWP::Protocol::https",
"Time::HiRes",
]:
env.safe_run("%s -i --notest --local-lib=%s '%s'" % (cpanm_cmd, env.system_install, perl_lib))
# Ensure paths we may have missed on install are accessible to regular user
if env.use_sudo:
paths = ["share", "share/java"]
for path in paths:
with quiet():
test_access = env.safe_run(
"test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)
)
if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _conda_cmd(env):
to_check = [os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda"]
for cmd in to_check:
with quiet():
test = env.safe_run("%s --version" % cmd)
if test.succeeded:
return cmd
return None
def _python_make(env):
with quiet():
full_pip = env.safe_run_output("which %s" % _pip_cmd(env))
run_cmd = env.safe_run if "/anaconda/" in full_pip else env.safe_sudo
# Clean up previously failed builds
env.safe_sudo("rm -rf /tmp/pip-build-%s" % env.user)
run_cmd("%s install --upgrade ." % full_pip)
for clean in ["dist", "build", "lib/*.egg-info"]:
env.safe_sudo("rm -rf %s" % clean)
def _install_brew_baseline(env, brew_cmd, ipkgs, packages):
"""Install baseline brew components not handled by dependency system.
- Installation of required Perl libraries.
- Ensures installed samtools does not overlap with bcftools
- Upgrades any package dependencies
"""
for dep in ["expat"]:
_install_pkg_latest(env, dep, brew_cmd, ipkgs)
# if installing samtools, avoid bcftools conflicts
if len([x for x in packages if x.find("samtools") >= 0]):
with settings(warn_only=True):
def _has_prog(prog):
try:
return int(
env.safe_run_output(
"{brew_cmd} list samtools | grep -c {prog}".format(
brew_cmd=brew_cmd, prog=prog)))
except ValueError:
return 0
if any(_has_prog(p) for p in ["bctools", "vcfutils.pl"]):
env.safe_run("{brew_cmd} uninstall {pkg}".format(
brew_cmd=brew_cmd, pkg="samtools"))
ipkgs["current"].pop("samtools", None)
_install_pkg_latest(env, "samtools", brew_cmd, ipkgs,
"--without-bcftools")
for dependency in ["htslib", "libmaus"]:
if dependency in packages:
if (dependency in ipkgs["outdated"]
or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"]
or dependency not in ipkgs["current"]):
_install_pkg_latest(env, dependency, brew_cmd, ipkgs)
if "cpanminus" in packages:
_install_pkg_latest(env, "cpanminus", brew_cmd, ipkgs)
cpanm_cmd = os.path.join(os.path.dirname(brew_cmd), "cpanm")
for perl_lib in [
"Statistics::Descriptive", "Archive::Extract", "Archive::Zip",
"Archive::Tar", "DBI", "LWP::Simple", "LWP::Protocol::https",
"Time::HiRes"
]:
env.safe_run("%s -i --notest --local-lib=%s '%s'" %
(cpanm_cmd, env.system_install, perl_lib))
# Ensure paths we may have missed on install are accessible to regular user
if env.use_sudo:
paths = ["share", "share/java"]
for path in paths:
with quiet():
test_access = env.safe_run(
"test -d %s/%s && test -O %s/%s" %
(env.system_install, path, env.system_install, path))
if test_access.failed and env.safe_exists(
"%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" %
(env.user, env.system_install, path))
def _conda_cmd(env):
to_check = [
os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda"
]
for cmd in to_check:
with quiet():
test = env.safe_run("%s --version" % cmd)
if test.succeeded:
return cmd
return None
def __work_dir():
work_dir = env.get("work_dir", None)
if not work_dir:
with quiet():
tmp_dir = env.safe_run_output("echo $TMPDIR")
if tmp_dir.failed or not tmp_dir.strip():
home_dir = env.safe_run_output("echo $HOME")
tmp_dir = os.path.join(home_dir, "tmp")
work_dir = os.path.join(tmp_dir.strip(), "cloudbiolinux")
return work_dir
def __work_dir():
work_dir = env.get("work_dir", None)
if not work_dir:
with quiet():
tmp_dir = env.safe_run_output("echo $TMPDIR")
if tmp_dir.failed or not tmp_dir.strip():
home_dir = env.safe_run_output("echo $HOME")
tmp_dir = os.path.join(home_dir, "tmp")
work_dir = os.path.join(tmp_dir.strip(), "cloudbiolinux")
return work_dir
def _install_brew_baseline(env, brew_cmd, ipkgs, packages):
"""Install baseline brew components not handled by dependency system.
- Installation of required Perl libraries.
- Upgrades any package dependencies
"""
for dep in ["openssl"]:
_safe_link_pkg(env, dep, brew_cmd)
for dep in ["expat", "pkg-config", "xz", "unzip"]:
_install_pkg(env, dep, brew_cmd, ipkgs)
# check if we have an older git and need to install it from brew
git_version = None
with quiet():
with settings(warn_only=True):
git_version = env.safe_run_output(
"git --version").strip().split()[-1]
if git_version and LooseVersion(git_version) < LooseVersion("1.7"):
_install_pkg(env, "git", brew_cmd, ipkgs)
for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba
env.safe_run("{brew_cmd} remove --force {dep}".format(**locals()))
for dependency in ["htslib"]:
if dependency in packages:
if (dependency in ipkgs["outdated"]
or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"]
or dependency not in ipkgs["current"]):
_install_pkg_latest(env, dependency, [], brew_cmd, ipkgs)
if "cpanminus" in packages:
_install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs)
_install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs)
cpan.install_packages(env)
# Ensure paths we may have missed on install are accessible to regular user
if env.use_sudo:
paths = ["share", "share/java"]
for path in paths:
with quiet():
test_access = env.safe_run(
"test -d %s/%s && test -O %s/%s" %
(env.system_install, path, env.system_install, path))
if test_access.failed and env.safe_exists(
"%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" %
(env.user, env.system_install, path))
def _get_current_pkgs(env, brew_cmd):
out = {}
with quiet():
which_out = env.safe_run_output("{brew_cmd} which".format(**locals()))
for line in which_out.split("\n"):
if line:
pkg, version = line.rstrip().split()
if pkg.endswith(":"):
pkg = pkg[:-1]
out[pkg] = version
return out
def _safe_update(env, brew_cmd, formula_repos, cur_taps):
"""Revert any taps if we fail to update due to local changes.
"""
with quiet():
with settings(warn_only=True):
out = env.safe_run("%s update" % brew_cmd)
if out.failed:
for repo in formula_repos:
if repo in cur_taps:
env.safe_run("%s untap %s" % (brew_cmd, repo))
env.safe_run("%s update" % brew_cmd)
def _conda_cmd(env):
if hasattr(env, "conda_cmd") and env.conda_cmd:
return env.conda_cmd
to_check = []
if env.hosts == ["localhost"]:
to_check.append(os.path.join(os.path.dirname(os.path.realpath(sys.executable)), "conda"))
to_check.extend([os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda"])
for cmd in to_check:
with quiet():
test = env.safe_run("%s --version" % cmd)
if test.succeeded:
return cmd
return None
def _get_current_pkgs(env, brew_cmd):
out = {}
with quiet():
which_out = env.safe_run_output("{brew_cmd} list --versions".format(**locals()))
for line in which_out.split("\n"):
if line:
parts = line.rstrip().split()
if len(parts) == 2:
pkg, version = line.rstrip().split()
if pkg.endswith(":"):
pkg = pkg[:-1]
out[pkg] = version
return out
def download(self, seq_dir):
zipped_file = None
genome_file = "%s.fa" % self._name
if not self._exists(genome_file, seq_dir):
prep_dir = "seq_prep"
env.safe_run("mkdir -p %s" % prep_dir)
with cd(prep_dir):
zipped_file = self._download_zip(seq_dir)
if zipped_file.endswith(".tar.gz"):
env.safe_run("tar -xzpf %s" % zipped_file)
elif zipped_file.endswith(".zip"):
env.safe_run("unzip %s" % zipped_file)
elif zipped_file.endswith(".gz"):
if not env.safe_exists("out.fa"):
env.safe_run("gunzip -c %s > out.fa" % zipped_file)
else:
raise ValueError("Do not know how to handle: %s" %
zipped_file)
tmp_file = genome_file.replace(".fa", ".txt")
result = env.safe_run_output("find `pwd` -name '*.fa'")
result = [x.strip() for x in result.split("\n")]
if len(result) == 1:
orig_result = result[0]
result = self._split_multifasta(result[0])
env.safe_run("rm %s" % orig_result)
result = self._karyotype_sort(result)
env.safe_run("rm -f inputs.txt")
for fname in result:
with quiet():
env.safe_run("echo '%s' >> inputs.txt" % fname)
env.safe_run("cat `cat inputs.txt` > %s" % (tmp_file))
for fname in result:
with quiet():
env.safe_run("rm -f %s" % fname)
env.safe_run("mv %s %s" % (tmp_file, genome_file))
zipped_file = os.path.join(prep_dir, zipped_file)
genome_file = os.path.join(prep_dir, genome_file)
return genome_file, [zipped_file]
def _install_brew_baseline(env, brew_cmd, ipkgs, packages):
"""Install baseline brew components not handled by dependency system.
- Installation of required Perl libraries.
- Upgrades any package dependencies
"""
for dep in ["openssl"]:
_safe_link_pkg(env, dep, brew_cmd)
for dep in ["expat", "pkg-config", "xz", "unzip"]:
_install_pkg(env, dep, brew_cmd, ipkgs)
# check if we have an older git and need to install it from brew
git_version = None
with quiet():
with settings(warn_only=True):
git_version = env.safe_run_output("git --version").strip().split()[-1]
if git_version and LooseVersion(git_version) < LooseVersion("1.7"):
_install_pkg(env, "git", brew_cmd, ipkgs)
for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba
env.safe_run("{brew_cmd} remove --force {dep}".format(**locals()))
for dependency in ["htslib"]:
if dependency in packages:
if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"]
or dependency not in ipkgs["current"]):
_install_pkg_latest(env, dependency, [], brew_cmd, ipkgs)
if "cpanminus" in packages:
_install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs)
_install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs)
cpan.install_packages(env)
# Ensure paths we may have missed on install are accessible to regular user
if env.use_sudo:
paths = ["share", "share/java"]
for path in paths:
with quiet():
test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path,
env.system_install, path))
if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def download(self, seq_dir):
zipped_file = None
genome_file = "%s.fa" % self._name
if not self._exists(genome_file, seq_dir):
prep_dir = "seq_prep"
env.safe_run("mkdir -p %s" % prep_dir)
with cd(prep_dir):
zipped_file = self._download_zip(seq_dir)
if zipped_file.endswith(".tar.gz"):
env.safe_run("tar -xzpf %s" % zipped_file)
elif zipped_file.endswith(".zip"):
env.safe_run("unzip %s" % zipped_file)
elif zipped_file.endswith(".gz"):
if not env.safe_exists("out.fa"):
env.safe_run("gunzip -c %s > out.fa" % zipped_file)
else:
raise ValueError("Do not know how to handle: %s" % zipped_file)
tmp_file = genome_file.replace(".fa", ".txt")
result = env.safe_run_output("find `pwd` -name '*.fa'")
result = [x.strip() for x in result.split("\n")]
if len(result) == 1:
orig_result = result[0]
result = self._split_multifasta(result[0])
env.safe_run("rm %s" % orig_result)
result = self._karyotype_sort(result)
env.safe_run("rm -f inputs.txt")
for fname in result:
with quiet():
env.safe_run("echo '%s' >> inputs.txt" % fname)
env.safe_run("cat `cat inputs.txt` > %s" % (tmp_file))
for fname in result:
with quiet():
env.safe_run("rm -f %s" % fname)
env.safe_run("mv %s %s" % (tmp_file, genome_file))
zipped_file = os.path.join(prep_dir, zipped_file)
genome_file = os.path.join(prep_dir, genome_file)
return genome_file, [zipped_file]
def _custom_unlink(env, brew_cmd, pkg):
"""Handle custom unlinking of packages that can break builds of others.
Does a temporary unlink and relink of packages while building.
"""
unlinks = {"lumpy-sv": ["bamtools"]}
for upkg in unlinks.get(pkg, []):
_safe_unlink_pkg(env, upkg, brew_cmd)
try:
yield None
finally:
for upkg in unlinks.get(pkg, []):
with settings(warn_only=True):
with quiet():
env.safe_run("%s link --overwrite %s" % (brew_cmd, upkg))
def _custom_unlink(env, brew_cmd, pkg):
"""Handle custom unlinking of packages that can break builds of others.
Does a temporary unlink and relink of packages while building.
"""
unlinks = {"lumpy-sv": ["bamtools"]}
for upkg in unlinks.get(pkg, []):
_safe_unlink_pkg(env, upkg, brew_cmd)
try:
yield None
finally:
for upkg in unlinks.get(pkg, []):
with settings(warn_only=True):
with quiet():
env.safe_run("%s link --overwrite %s" % (brew_cmd, upkg))
def _symlink_bin(package, env, conda_info, conda_bin, files=None, prefix=""):
"""Symlink files installed in the bin directory into install directory.
"""
package = package.split("=")[0]
final_bindir = os.path.join(env.system_install, "bin")
for pkg_subdir in json.loads(env.safe_run_output("{conda_bin} list --json -f {package}".format(**locals()))):
for pkg_dir in conda_info["pkgs_dirs"]:
pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin")
if env.safe_exists(pkg_bindir):
if not files:
with quiet():
files = env.safe_run_output("ls -1 {pkg_bindir}".format(**locals())).split()
for fname in files:
_do_symlink(os.path.join(pkg_bindir, fname),
os.path.join(final_bindir, "%s%s" % (prefix, fname)))
def _pip_cmd(env):
"""Retrieve pip command for installing python packages, allowing configuration.
"""
to_check = ["pip"]
if "pip_cmd" in env and env.pip_cmd:
to_check.append(env.pip_cmd)
if not env.use_sudo:
to_check.append(os.path.join(env.system_install, "bin", "pip"))
if "python_version_ext" in env and env.python_version_ext:
to_check.append("pip-{0}".format(env.python_version_ext))
for cmd in to_check:
with quiet():
pip_version = env.safe_run("%s --version" % cmd)
if pip_version.succeeded:
return cmd
raise ValueError("Could not find pip installer from: %s" % to_check)
def up_to_date(env, cmd, version, args=None, stdout_flag=None,
stdout_index=-1):
"""Check if the given command is up to date with the provided version.
"""
if shared._executable_not_on_path(cmd):
return False
if args:
cmd = cmd + " " + " ".join(args)
with quiet():
path_safe = "export PATH=$PATH:%s/bin && " % env.system_install
out = env.safe_run_output(path_safe + cmd)
if stdout_flag:
iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index)
else:
iversion = out.strip()
iversion = _clean_version(iversion)
return LooseVersion(iversion) >= LooseVersion(version)
def _safe_update(env, brew_cmd, formula_repos, cur_taps):
"""Revert any taps if we fail to update due to local changes.
"""
with _git_stash(env, brew_cmd):
with quiet():
with settings(warn_only=True):
out = env.safe_run("%s update" % brew_cmd)
if out.failed:
for repo in formula_repos:
if repo in cur_taps:
env.safe_run("%s untap %s" % (brew_cmd, repo))
with settings(warn_only=True):
out = env.safe_run("%s update" % brew_cmd)
if out.failed:
print("\n\nHomebrew update failed.")
print("You might need to upgrade git by installing inside bcbio with:")
print("'brew install git --env=inherit --ignore-dependences'\n\n")
def _conda_cmd(env):
if hasattr(env, "conda_cmd") and env.conda_cmd:
return env.conda_cmd
to_check = []
if env.hosts == ["localhost"]:
to_check.append(
os.path.join(os.path.dirname(os.path.realpath(sys.executable)),
"conda"))
to_check.extend([
os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda"
])
for cmd in to_check:
with quiet():
test = env.safe_run("%s --version" % cmd)
if test.succeeded:
return cmd
return None
def _safe_update(env, brew_cmd, formula_repos, cur_taps):
"""Revert any taps if we fail to update due to local changes.
"""
with _git_stash(env, brew_cmd):
with quiet():
with settings(warn_only=True):
out = env.safe_run("%s update" % brew_cmd)
if out.failed:
for repo in formula_repos:
if repo in cur_taps:
env.safe_run("%s untap %s" % (brew_cmd, repo))
with settings(warn_only=True):
out = env.safe_run("%s update" % brew_cmd)
if out.failed:
print("\n\nHomebrew update failed.")
print("You might need to upgrade git by installing inside bcbio with:")
print("'brew install git --env=inherit --ignore-dependences'\n\n")
def _symlink_bin(package, env, conda_info, conda_bin, files=None, prefix=""):
"""Symlink files installed in the bin directory into install directory.
"""
package = package.split("=")[0]
final_bindir = os.path.join(env.system_install, "bin")
for pkg_subdir in json.loads(
env.safe_run_output(
"{conda_bin} list --json -f {package}".format(**locals()))):
for pkg_dir in conda_info["pkgs_dirs"]:
pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin")
if env.safe_exists(pkg_bindir):
if not files:
with quiet():
files = env.safe_run_output(
"ls -1 {pkg_bindir}".format(**locals())).split()
for fname in files:
_do_symlink(
os.path.join(pkg_bindir, fname),
os.path.join(final_bindir, "%s%s" % (prefix, fname)))
def get_installed_version(env, cmd, version, args=None, stdout_flag=None,
stdout_index=-1):
"""Check if the given command is up to date with the provided version.
"""
if shared._executable_not_on_path(cmd):
return False
if args:
cmd = cmd + " " + " ".join(args)
with quiet():
path_safe = ("export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:{s}/lib/pkgconfig && "
"export PATH=$PATH:{s}/bin && "
"export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{s}/lib && ".format(s=env.system_install))
out = env.safe_run_output(path_safe + cmd)
if stdout_flag:
iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index)
else:
iversion = out.strip()
iversion = _clean_version(iversion)
return iversion
def _pip_cmd(env):
"""Retrieve pip command for installing python packages, allowing configuration.
"""
anaconda_pip = os.path.join(env.system_install, "anaconda", "bin", "pip")
if env.safe_exists(anaconda_pip):
to_check = [anaconda_pip]
else:
to_check = ["pip"]
if "pip_cmd" in env and env.pip_cmd:
to_check.append(env.pip_cmd)
if not env.use_sudo:
to_check.append(os.path.join(env.system_install, "bin", "pip"))
if "python_version_ext" in env and env.python_version_ext:
to_check.append("pip-{0}".format(env.python_version_ext))
for cmd in to_check:
with quiet():
pip_version = env.safe_run("%s --version" % cmd)
if pip_version.succeeded:
return cmd
raise ValueError("Could not find pip installer from: %s" % to_check)
def up_to_date(env,
cmd,
version,
args=None,
stdout_flag=None,
stdout_index=-1):
"""Check if the given command is up to date with the provided version.
"""
if shared._executable_not_on_path(cmd):
return False
if args:
cmd = cmd + " " + " ".join(args)
with quiet():
path_safe = "export PATH=$PATH:%s/bin && "
out = env.safe_run_output(path_safe + cmd)
if stdout_flag:
iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index)
else:
iversion = out.strip()
iversion = _clean_version(iversion)
return LooseVersion(iversion) >= LooseVersion(version)
def _determine_distribution(env):
"""
Attempt to automatically determine the distribution of the target machine.
Currently works for Ubuntu, CentOS, Debian, Scientific Linux and Mac OS X.
"""
with quiet():
output = env.safe_run_output("cat /etc/*release").lower()
if output.find("id=ubuntu") >= 0:
return "ubuntu"
elif output.find("centos release") >= 0:
return "centos"
elif output.find("centos linux release") >= 0:
return "centos"
elif output.find("red hat enterprise linux") >= 0:
return "centos"
elif output.find("fedora") >= 0:
return "centos"
# Amazon AMIs are Red-Hat based
elif output.find("amzn") >= 0 or output.find("amazon") >= 0:
return "centos"
elif output.find("suse linux") >= 0:
return "suse"
elif output.find("opensuse") >= 0:
return "suse"
elif output.find("scientific linux") >= 0:
return "scientificlinux"
elif env.safe_exists("/etc/debian_version"):
return "debian"
elif output.find("id=arch") >= 0 or output.find('id_like="arch"') >= 0:
return "arch"
elif output.find("antergos") >= 0:
return "arch"
# check for file used by Python's platform.mac_ver
elif env.safe_exists("/System/Library/CoreServices/SystemVersion.plist"):
return "macosx"
else:
raise Exception(
"Attempt to automatically determine Linux distribution of target machine failed:\n%s"
% output)
def _install_brew_baseline(env, brew_cmd, ipkgs, packages):
"""Install baseline brew components not handled by dependency system.
- Installation of required Perl libraries.
- Ensures installed samtools does not overlap with bcftools
- Upgrades any package dependencies
"""
for dep in ["cpanminus", "expat"]:
_install_pkg_latest(env, dep, brew_cmd, ipkgs)
# if installing samtools, avoid bcftools conflicts
if len([x for x in packages if x.find("samtools") >= 0]):
with settings(warn_only=True):
try:
has_bcftools = int(env.safe_run_output("{brew_cmd} list samtools | grep -c bcftools".format(
brew_cmd=brew_cmd)))
except ValueError:
has_bcftools = 0
if has_bcftools:
env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools"))
ipkgs["current"].pop("samtools", None)
_install_pkg_latest(env, "samtools", brew_cmd, ipkgs, "--without-bcftools")
for dependency in ["htslib", "libmaus"]:
if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"]
or dependency not in ipkgs["current"]):
_install_pkg_latest(env, dependency, brew_cmd, ipkgs)
cpanm_cmd = os.path.join(os.path.dirname(brew_cmd), "cpanm")
for perl_lib in ["Statistics::Descriptive"]:
env.safe_run("%s -i --notest --local-lib=%s '%s'" % (cpanm_cmd, env.system_install, perl_lib))
# Ensure paths we may have missed on install are accessible to regular user
if env.use_sudo:
paths = ["share", "share/java"]
for path in paths:
with quiet():
test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path,
env.system_install, path))
if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)):
env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _safe_uninstall_pkg(env, pkg_str, brew_cmd):
"""Uninstall packages which get pulled in even when unlinked by brew.
"""
with settings(warn_only=True):
with quiet():
env.safe_run("{brew_cmd} uninstall {pkg_str}".format(**locals()))
def install_homebrew(env):
"""Homebrew package manager for OSX and Linuxbrew for linux systems.
https://github.com/mxcl/homebrew
https://github.com/Homebrew/linuxbrew
"""
if env.distribution == "macosx":
with quiet():
test_brewcmd = env.safe_run("brew --version")
if not test_brewcmd.succeeded:
env.safe_run(
'ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/go/install)"'
)
else:
brewcmd = os.path.join(env.system_install, "bin", "brew")
with quiet():
test_brewcmd = env.safe_run("%s --version" % brewcmd)
if not test_brewcmd.succeeded or _linuxbrew_origin_problem(brewcmd):
with shared._make_tmp_dir() as tmp_dir:
with cd(tmp_dir):
if env.safe_exists("linuxbrew"):
env.safe_run("rm -rf linuxbrew")
for cleandir in ["Library", ".git"]:
if env.safe_exists("%s/%s" %
(env.system_install, cleandir)):
env.safe_run("rm -rf %s/%s" %
(env.system_install, cleandir))
env.safe_run(
"git clone https://github.com/Homebrew/linuxbrew.git")
with cd("linuxbrew"):
if not env.safe_exists(env.system_install):
env.safe_sudo("mkdir -p %s" % env.system_install)
env.safe_sudo("chown %s %s" %
(env.user, env.system_install))
paths = [
"bin", "etc", "include", "lib", "lib/pkgconfig",
"Library", "sbin", "share", "var", "var/log",
"share/java", "share/locale", "share/man",
"share/man/man1", "share/man/man2",
"share/man/man3", "share/man/man4",
"share/man/man5", "share/man/man6",
"share/man/man7", "share/man/man8", "share/info",
"share/doc", "share/aclocal",
"lib/python2.7/site-packages",
"lib/python2.6/site-packages",
"lib/python3.2/site-packages",
"lib/python3.3/site-packages", "lib/perl5",
"lib/perl5/site_perl"
]
if not env.safe_exists("%s/bin" % env.system_install):
env.safe_sudo("mkdir -p %s/bin" %
env.system_install)
for path in paths:
if env.safe_exists("%s/%s" %
(env.system_install, path)):
env.safe_sudo(
"chown %s %s/%s" %
(env.user, env.system_install, path))
if not env.safe_exists(
"%s/Library" % env.system_install):
env.safe_run("mv Library %s" % env.system_install)
if not env.safe_exists("%s/.git" % env.system_install):
env.safe_run("mv .git %s" % env.system_install)
man_dir = "share/man/man1"
if not env.safe_exists("%s/%s" %
(env.system_install, man_dir)):
env.safe_run("mkdir -p %s/%s" %
(env.system_install, man_dir))
env.safe_run("mv -f %s/brew.1 %s/%s" %
(man_dir, env.system_install, man_dir))
env.safe_run("mv -f bin/brew %s/bin" %
env.system_install)
def _safe_uninstall_pkg(env, pkg_str, brew_cmd):
"""Uninstall packages which get pulled in even when unlinked by brew.
"""
with settings(warn_only=True):
with quiet():
env.safe_run("{brew_cmd} uninstall {pkg_str}".format(**locals()))
def _safe_link_pkg(env, pkg_str, brew_cmd):
"""Link packages required for builds, but not necessarily installed
"""
with settings(warn_only=True):
with quiet():
env.safe_run("{brew_cmd} link --overwrite {pkg_str}".format(**locals()))
def _safe_unlink_pkg(env, pkg_str, brew_cmd):
"""Unlink packages which can cause issues with a Linux system.
"""
with settings(warn_only=True):
with quiet():
env.safe_run("{brew_cmd} unlink {pkg_str}".format(**locals()))
def _safe_unlink_pkg(env, pkg_str, brew_cmd):
"""Unlink packages which can cause issues with a Linux system.
"""
with settings(warn_only=True):
with quiet():
env.safe_run("{brew_cmd} unlink {pkg_str}".format(**locals()))
