def _link_bin(package, env, conda_info, conda_bin, files=None, prefix=""): """Link files installed in the bin directory into the install directory. This is imperfect but we're trying not to require injecting everything in the anaconda directory into a user's path. """ package = package.split("=")[0] final_bindir = os.path.join(env.system_install, "bin") base_bindir = os.path.dirname(conda_bin) # resolve any symlinks in the final and base heirarchies with quiet(): final_bindir = env.safe_run_output("cd %s && pwd -P" % final_bindir) base_bindir = env.safe_run_output("cd %s && pwd -P" % base_bindir) for pkg_subdir in json.loads( env.safe_run_output( "{conda_bin} list --json -f {package}".format(**locals()))): pkg_subdir = pkg_subdir.split("::")[-1] for pkg_dir in conda_info["pkgs_dirs"]: pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin") if env.safe_exists(pkg_bindir): if not files: with quiet(): files = env.safe_run_output( "ls -1 {pkg_bindir}".format(**locals())).split() for fname in files: # symlink to the original file in the /anaconda/bin directory # this could be a hard or soft link base_fname = os.path.join(base_bindir, fname) if os.path.exists(base_fname) and os.path.lexists( base_fname): _do_link( base_fname, os.path.join(final_bindir, "%s%s" % (prefix, fname)))
def _link_bin(package, env, conda_info, conda_bin, files=None, prefix=""): """Link files installed in the bin directory into the install directory. This is imperfect but we're trying not to require injecting everything in the anaconda directory into a user's path. """ package = package.split("=")[0] final_bindir = os.path.join(env.system_install, "bin") base_bindir = os.path.dirname(conda_bin) # resolve any symlinks in the final and base heirarchies with quiet(): final_bindir = env.safe_run_output("cd %s && pwd -P" % final_bindir) base_bindir = env.safe_run_output("cd %s && pwd -P" % base_bindir) for pkg_subdir in json.loads(env.safe_run_output("{conda_bin} list --json -f {package}".format(**locals()))): for pkg_dir in conda_info["pkgs_dirs"]: pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin") if env.safe_exists(pkg_bindir): if not files: with quiet(): files = env.safe_run_output("ls -1 {pkg_bindir}".format(**locals())).split() for fname in files: # symlink to the original file in the /anaconda/bin directory # this could be a hard or soft link base_fname = os.path.join(base_bindir, fname) if os.path.exists(base_fname) and os.path.lexists(base_fname): _do_link(base_fname, os.path.join(final_bindir, "%s%s" % (prefix, fname)))
def _is_anaconda(env): """Check if we have a conda command or are in an anaconda subdirectory. """ with quiet(): conda = _conda_cmd(env) has_conda = conda and env.safe_run_output("%s -h" % conda).startswith("usage: conda") with quiet(): full_pip = env.safe_run_output("which %s" % _pip_cmd(env)) in_anaconda_dir = "/anaconda/" in full_pip return has_conda or in_anaconda_dir
def _is_anaconda(env): """Check if we have a conda command or are in an anaconda subdirectory. """ with quiet(): conda = _conda_cmd(env) has_conda = conda and env.safe_run_output( "%s -h" % conda).startswith("usage: conda") with quiet(): full_pip = env.safe_run_output("which %s" % _pip_cmd(env)) in_anaconda_dir = "/anaconda/" in full_pip return has_conda or in_anaconda_dir
def _is_anaconda(env): """Check if we have a conda command or are in an anaconda subdirectory. """ with quiet(): conda = _conda_cmd(env) has_conda = conda and env.safe_run_output("%s -h" % conda).startswith("usage: conda") with quiet(): try: full_pip = env.safe_run_output("which %s" % _pip_cmd(env)) except ValueError: full_pip = None in_anaconda_dir = full_pip and full_pip.succeeded and "/anaconda/" in full_pip return has_conda or in_anaconda_dir
def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. - Installation of required Perl libraries. - Ensures installed samtools does not overlap with bcftools - Upgrades any package dependencies """ for dep in ["openssl"]: _safe_link_pkg(env, dep, brew_cmd) for dep in ["expat", "pkg-config", "xz"]: _install_pkg(env, dep, brew_cmd, ipkgs) # check if we have an older git and need to install it from brew git_version = None with quiet(): with settings(warn_only=True): git_version = env.safe_run_output("git --version").strip().split()[-1] if git_version and LooseVersion(git_version) < LooseVersion("1.7"): _install_pkg(env, "git", brew_cmd, ipkgs) for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba env.safe_run("{brew_cmd} remove --force {dep}".format(**locals())) # if installing samtools, avoid bcftools conflicts if len([x for x in packages if x.find("samtools") >= 0]): with settings(warn_only=True): def _has_prog(prog): try: return int(env.safe_run_output("{brew_cmd} list samtools | grep -c {prog} | cat".format( brew_cmd=brew_cmd, prog=prog))) except ValueError: return 0 if any(_has_prog(p) for p in ["bctools", "vcfutils.pl"]): env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools")) ipkgs["current"].pop("samtools", None) _install_pkg_latest(env, "samtools", ["--without-curses"], brew_cmd, ipkgs) for dependency in ["htslib"]: if dependency in packages: if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"] or dependency not in ipkgs["current"]): _install_pkg_latest(env, dependency, [], brew_cmd, ipkgs) if "cpanminus" in packages: _install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs) _install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs) cpan.install_packages(env) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)) if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. - Installation of required Perl libraries. - Ensures installed samtools does not overlap with bcftools - Upgrades any package dependencies """ for dep in ["openssl"]: _safe_link_pkg(env, dep, brew_cmd) for dep in ["expat", "pkg-config", "cmake", "xz"]: _install_pkg(env, dep, brew_cmd, ipkgs) # check if we have an older git and need to install it from brew git_version = None with quiet(): with settings(warn_only=True): git_version = env.safe_run_output("git --version").strip().split()[-1] if git_version and LooseVersion(git_version) < LooseVersion("1.7"): _install_pkg(env, "git", brew_cmd, ipkgs) for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba env.safe_run("{brew_cmd} remove --force {dep}".format(**locals())) # if installing samtools, avoid bcftools conflicts if len([x for x in packages if x.find("samtools") >= 0]): with settings(warn_only=True): def _has_prog(prog): try: return int(env.safe_run_output("{brew_cmd} list samtools | grep -c {prog} | cat".format( brew_cmd=brew_cmd, prog=prog))) except ValueError: return 0 if any(_has_prog(p) for p in ["bctools", "vcfutils.pl"]): env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools")) ipkgs["current"].pop("samtools", None) _install_pkg_latest(env, "samtools", ["--without-curses"], brew_cmd, ipkgs) for dependency in ["htslib"]: if dependency in packages: if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"] or dependency not in ipkgs["current"]): _install_pkg_latest(env, dependency, [], brew_cmd, ipkgs) if "cpanminus" in packages: _install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs) _install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs) cpan.install_packages(env) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)) if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def install_homebrew(env): """Homebrew package manager for OSX and Linuxbrew for linux systems. https://github.com/mxcl/homebrew https://github.com/Homebrew/linuxbrew """ if env.distribution == "macosx": with quiet(): test_brewcmd = env.safe_run("brew --version") if not test_brewcmd.succeeded: env.safe_run('ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/go/install)"') else: brewcmd = os.path.join(env.system_install, "bin", "brew") with quiet(): test_brewcmd = env.safe_run("%s --version" % brewcmd) if not test_brewcmd.succeeded or _linuxbrew_origin_problem(brewcmd): with shared._make_tmp_dir() as tmp_dir: with cd(tmp_dir): if env.safe_exists("linuxbrew"): env.safe_run("rm -rf linuxbrew") for cleandir in ["Library", ".git"]: if env.safe_exists("%s/%s" % (env.system_install, cleandir)): env.safe_run("rm -rf %s/%s" % (env.system_install, cleandir)) env.safe_run("git clone https://github.com/Homebrew/linuxbrew.git") with cd("linuxbrew"): if not env.safe_exists(env.system_install): env.safe_sudo("mkdir -p %s" % env.system_install) env.safe_sudo("chown %s %s" % (env.user, env.system_install)) paths = ["bin", "etc", "include", "lib", "lib/pkgconfig", "Library", "sbin", "share", "var", "var/log", "share/java", "share/locale", "share/man", "share/man/man1", "share/man/man2", "share/man/man3", "share/man/man4", "share/man/man5", "share/man/man6", "share/man/man7", "share/man/man8", "share/info", "share/doc", "share/aclocal", "lib/python2.7/site-packages", "lib/python2.6/site-packages", "lib/python3.2/site-packages", "lib/python3.3/site-packages", "lib/perl5", "lib/perl5/site_perl"] if not env.safe_exists("%s/bin" % env.system_install): env.safe_sudo("mkdir -p %s/bin" % env.system_install) for path in paths: if env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path)) if not env.safe_exists("%s/Library" % env.system_install): env.safe_run("mv Library %s" % env.system_install) if not env.safe_exists("%s/.git" % env.system_install): env.safe_run("mv .git %s" % env.system_install) man_dir = "share/man/man1" if not env.safe_exists("%s/%s" % (env.system_install, man_dir)): env.safe_run("mkdir -p %s/%s" % (env.system_install, man_dir)) env.safe_run("mv -f %s/brew.1 %s/%s" % (man_dir, env.system_install, man_dir)) env.safe_run("mv -f bin/brew %s/bin" % env.system_install)
def _is_anaconda(env): """Check if we have a conda command or are in an anaconda subdirectory. """ with quiet(): conda = _conda_cmd(env) has_conda = conda and env.safe_run_output( "%s -h" % conda).startswith("usage: conda") with quiet(): try: full_pip = env.safe_run_output("which %s" % _pip_cmd(env)) except ValueError: full_pip = None in_anaconda_dir = full_pip and full_pip.succeeded and "/anaconda/" in full_pip return has_conda or in_anaconda_dir
def _setup_deb_general(): """Shared settings for different debian based/derived distributions. """ env.logger.debug("Debian-shared setup") env.sources_file = "/etc/apt/sources.list.d/cloudbiolinux.list" env.global_sources_file = "/etc/apt/sources.list" env.apt_preferences_file = "/etc/apt/preferences" if not hasattr(env, "python_version_ext"): env.python_version_ext = "" if not hasattr(env, "ruby_version_ext"): env.ruby_version_ext = "1.9.1" if not env.has_key("java_home"): # Try to determine java location from update-alternatives java_home = "/usr/lib/jvm/java-7-openjdk-amd64" with quiet(): java_info = env.safe_run_output( "update-alternatives --display java") for line in java_info.split("\n"): if line.strip().startswith("link currently points to"): java_home = line.split()[-1].strip() java_home = java_home.replace("/jre/bin/java", "") env.java_home = java_home shared_sources = [ "deb http://nebc.nerc.ac.uk/bio-linux/ unstable bio-linux", # Bio-Linux "deb http://download.virtualbox.org/virtualbox/debian %s contrib", # virtualbox ] return shared_sources
def _determine_distribution(env): """ Attempt to automatically determine the distribution of the target machine. Currently works for Ubuntu, CentOS, Debian, Scientific Linux and Mac OS X. """ with quiet(): output = env.safe_run_output("cat /etc/*release").lower() if output.find("distrib_id=ubuntu") >= 0: return "ubuntu" elif output.find("centos release") >= 0: return "centos" elif output.find("red hat enterprise linux server release") >= 0: return "centos" elif output.find("scientific linux release") >= 0: return "scientificlinux" elif env.safe_exists("/etc/debian_version"): return "debian" # check for file used by Python's platform.mac_ver elif env.safe_exists("/System/Library/CoreServices/SystemVersion.plist"): return "macosx" else: raise Exception( "Attempt to automatically determine Linux distribution of target machine failed, please manually specify distribution in fabricrc.txt" )
def _install_bottle(env, brew_cmd, pkg, ipkgs): """Install Linux bottles for brew packages that can be tricky to build. """ if env.distribution == "macosx": # Only Linux bottles, build away on Mac return False pkg_version, is_linked = _latest_pkg_version(env, brew_cmd, pkg) install_version = ipkgs["current"].get(pkg) if pkg_version == install_version: # Up to date if not is_linked: env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg)) return True elif install_version or pkg in ipkgs["outdated"]: env.safe_run("{brew_cmd} remove --force {pkg}".format(**locals())) url = BOTTLE_URL.format(pkg=pkg, version=pkg_version) brew_cachedir = env.safe_run_output("%s --cache" % brew_cmd) brew_cellar = os.path.join(env.safe_run_output("%s --prefix" % brew_cmd), "Cellar") with quiet(): env.safe_run("mkdir -p %s" % brew_cellar) out_file = os.path.join(brew_cachedir, os.path.basename(url)) if env.safe_exists(out_file): env.safe_run("rm -f %s" % out_file) bottle_file = shared._remote_fetch(env, url, out_file=out_file, allow_fail=True, samedir=True) if bottle_file: with cd(brew_cellar): env.safe_run("tar -xf %s" % bottle_file) env.safe_run("%s link --overwrite %s" % (brew_cmd, pkg)) return True else: return False
def _determine_distribution(env): """ Attempt to automatically determine the distribution of the target machine. Currently works for Ubuntu, CentOS, Debian, Scientific Linux and Mac OS X. """ with quiet(): output = env.safe_run_output("cat /etc/*release").lower() if output.find("distrib_id=ubuntu") >= 0: return "ubuntu" elif output.find("centos release") >= 0: return "centos" elif output.find("centos linux release") >= 0: return "centos" elif output.find("red hat") >= 0: return "centos" elif output.find("fedora release") >= 0: return "centos" elif output.find("amzn") >= 0: # Amazon AMIs are Red-Hat based return "centos" elif output.find("suse linux") >= 0: return "suse" elif output.find("opensuse") >= 0: return "suse" elif output.find("scientific linux") >= 0: return "scientificlinux" elif env.safe_exists("/etc/debian_version"): return "debian" elif output.find("id=arch") >= 0: return "arch" # check for file used by Python's platform.mac_ver elif env.safe_exists("/System/Library/CoreServices/SystemVersion.plist"): return "macosx" else: raise Exception("Attempt to automatically determine Linux distribution of target machine failed, please manually specify distribution in fabricrc.txt")
def get_installed_version(env, cmd, version, args=None, stdout_flag=None, stdout_index=-1): """Check if the given command is up to date with the provided version. """ if shared._executable_not_on_path(cmd): return False if args: cmd = cmd + " " + " ".join(args) with quiet(): path_safe = ( "export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:{s}/lib/pkgconfig && " "export PATH=$PATH:{s}/bin && " "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{s}/lib && ".format( s=env.system_install)) out = env.safe_run_output(path_safe + cmd) if stdout_flag: iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index) else: iversion = out.strip() iversion = _clean_version(iversion) if " not found in the pkg-config search path" in iversion: return False return iversion
def _git_stash(env, brew_cmd): """Perform a safe git stash around an update. This circumvents brews internal stash approach which doesn't work on older versions of git and is sensitive to missing config.emails. """ brew_prefix = env.safe_run_output("{brew_cmd} --prefix".format(**locals())) with cd(brew_prefix): with quiet(): with settings(warn_only=True): env.safe_run("git config user.email '*****@*****.**'") check_diff = env.safe_run("git diff --quiet") git_version = env.safe_run_output("git --version").strip().split()[-1] if git_version and LooseVersion(git_version) < LooseVersion("1.7"): if check_diff.return_code > 0: with cd(brew_prefix): with settings(warn_only=True): env.safe_run("git stash --quiet") try: yield None finally: if check_diff.return_code > 0: with cd(brew_prefix): with settings(warn_only=True): env.safe_run("git stash pop --quiet") else: yield None
def _safe_link_pkg(env, pkg_str, brew_cmd): """Link packages required for builds, but not necessarily installed """ with settings(warn_only=True): with quiet(): env.safe_run( "{brew_cmd} link --overwrite {pkg_str}".format(**locals()))
def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. Handles installation of required Perl libraries. """ for dep in ["cpanminus", "expat"]: _install_pkg_latest(env, dep, brew_cmd, ipkgs) # if installing samtools, avoid conflicts with cbl and homebrew-science versions if len([x for x in packages if x.find("samtools") >= 0]): with settings(warn_only=True): try: has_bcftools = int(env.safe_run_output("{brew_cmd} list samtools | grep -c bcftools".format( brew_cmd=brew_cmd))) except: has_bcftools = int(env.safe_run_output("{brew_cmd} list samtools | grep -c bcftools".format( brew_cmd=brew_cmd)).split("\n")[-1].rstrip()) if has_bcftools: env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools")) cpanm_cmd = os.path.join(os.path.dirname(brew_cmd), "cpanm") for perl_lib in ["Statistics::Descriptive"]: env.safe_run("%s -i --notest --local-lib=%s '%s'" % (cpanm_cmd, env.system_install, perl_lib)) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)) if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _setup_deb_general(): """Shared settings for different debian based/derived distributions. """ env.logger.debug("Debian-shared setup") env.sources_file = "/etc/apt/sources.list.d/cloudbiolinux.list" env.global_sources_file = "/etc/apt/sources.list" env.apt_preferences_file = "/etc/apt/preferences" if not hasattr(env, "python_version_ext"): env.python_version_ext = "" if not hasattr(env, "ruby_version_ext"): env.ruby_version_ext = "1.9.1" if not env.has_key("java_home"): # Try to determine java location from update-alternatives java_home = "/usr/lib/jvm/java-7-openjdk-amd64" with quiet(): java_info = env.safe_run_output("update-alternatives --display java") for line in java_info.split("\n"): if line.strip().startswith("link currently points to"): java_home = line.split()[-1].strip() java_home = java_home.replace("/jre/bin/java", "") env.java_home = java_home shared_sources = [ "deb http://download.virtualbox.org/virtualbox/debian %s contrib", # virtualbox ] return shared_sources
def _git_stash(env, brew_cmd): """Perform a safe git stash around an update. This circumvents brews internal stash approach which doesn't work on older versions of git and is sensitive to missing config.emails. """ brew_prefix = env.safe_run_output("{brew_cmd} --prefix".format(**locals())) with cd(brew_prefix): with quiet(): with settings(warn_only=True): env.safe_run("git config user.email '*****@*****.**'") check_diff = env.safe_run("git diff --quiet") git_version = env.safe_run_output( "git --version").strip().split()[-1] if git_version and LooseVersion(git_version) < LooseVersion("1.7"): if check_diff.return_code > 0: with cd(brew_prefix): with settings(warn_only=True): env.safe_run("git stash --quiet") try: yield None finally: if check_diff.return_code > 0: with cd(brew_prefix): with settings(warn_only=True): env.safe_run("git stash pop --quiet") else: yield None
def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. - Installation of required Perl libraries. - Ensures installed samtools does not overlap with bcftools - Upgrades any package dependencies """ for dep in ["expat"]: _install_pkg_latest(env, dep, [], brew_cmd, ipkgs) # if installing samtools, avoid bcftools conflicts if len([x for x in packages if x.find("samtools") >= 0]): with settings(warn_only=True): def _has_prog(prog): try: return int( env.safe_run_output( "{brew_cmd} list samtools | grep -c {prog}".format(brew_cmd=brew_cmd, prog=prog) ) ) except ValueError: return 0 if any(_has_prog(p) for p in ["bctools", "vcfutils.pl"]): env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools")) ipkgs["current"].pop("samtools", None) _install_pkg_latest(env, "samtools", ["--devel"], brew_cmd, ipkgs) for dependency in ["htslib", "libmaus", "cmake"]: if dependency in packages: if ( dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"] or dependency not in ipkgs["current"] ): _install_pkg_latest(env, dependency, [], brew_cmd, ipkgs) if "cpanminus" in packages: _install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs) cpanm_cmd = os.path.join(os.path.dirname(brew_cmd), "cpanm") for perl_lib in [ "Statistics::Descriptive", "Archive::Extract", "Archive::Zip", "Archive::Tar", "DBI", "LWP::Simple", "LWP::Protocol::https", "Time::HiRes", ]: env.safe_run("%s -i --notest --local-lib=%s '%s'" % (cpanm_cmd, env.system_install, perl_lib)) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run( "test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path) ) if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _conda_cmd(env): to_check = [os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda"] for cmd in to_check: with quiet(): test = env.safe_run("%s --version" % cmd) if test.succeeded: return cmd return None
def _python_make(env): with quiet(): full_pip = env.safe_run_output("which %s" % _pip_cmd(env)) run_cmd = env.safe_run if "/anaconda/" in full_pip else env.safe_sudo # Clean up previously failed builds env.safe_sudo("rm -rf /tmp/pip-build-%s" % env.user) run_cmd("%s install --upgrade ." % full_pip) for clean in ["dist", "build", "lib/*.egg-info"]: env.safe_sudo("rm -rf %s" % clean)
def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. - Installation of required Perl libraries. - Ensures installed samtools does not overlap with bcftools - Upgrades any package dependencies """ for dep in ["expat"]: _install_pkg_latest(env, dep, brew_cmd, ipkgs) # if installing samtools, avoid bcftools conflicts if len([x for x in packages if x.find("samtools") >= 0]): with settings(warn_only=True): def _has_prog(prog): try: return int( env.safe_run_output( "{brew_cmd} list samtools | grep -c {prog}".format( brew_cmd=brew_cmd, prog=prog))) except ValueError: return 0 if any(_has_prog(p) for p in ["bctools", "vcfutils.pl"]): env.safe_run("{brew_cmd} uninstall {pkg}".format( brew_cmd=brew_cmd, pkg="samtools")) ipkgs["current"].pop("samtools", None) _install_pkg_latest(env, "samtools", brew_cmd, ipkgs, "--without-bcftools") for dependency in ["htslib", "libmaus"]: if dependency in packages: if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"] or dependency not in ipkgs["current"]): _install_pkg_latest(env, dependency, brew_cmd, ipkgs) if "cpanminus" in packages: _install_pkg_latest(env, "cpanminus", brew_cmd, ipkgs) cpanm_cmd = os.path.join(os.path.dirname(brew_cmd), "cpanm") for perl_lib in [ "Statistics::Descriptive", "Archive::Extract", "Archive::Zip", "Archive::Tar", "DBI", "LWP::Simple", "LWP::Protocol::https", "Time::HiRes" ]: env.safe_run("%s -i --notest --local-lib=%s '%s'" % (cpanm_cmd, env.system_install, perl_lib)) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run( "test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)) if test_access.failed and env.safe_exists( "%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _conda_cmd(env): to_check = [ os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda" ] for cmd in to_check: with quiet(): test = env.safe_run("%s --version" % cmd) if test.succeeded: return cmd return None
def __work_dir(): work_dir = env.get("work_dir", None) if not work_dir: with quiet(): tmp_dir = env.safe_run_output("echo $TMPDIR") if tmp_dir.failed or not tmp_dir.strip(): home_dir = env.safe_run_output("echo $HOME") tmp_dir = os.path.join(home_dir, "tmp") work_dir = os.path.join(tmp_dir.strip(), "cloudbiolinux") return work_dir
def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. - Installation of required Perl libraries. - Upgrades any package dependencies """ for dep in ["openssl"]: _safe_link_pkg(env, dep, brew_cmd) for dep in ["expat", "pkg-config", "xz", "unzip"]: _install_pkg(env, dep, brew_cmd, ipkgs) # check if we have an older git and need to install it from brew git_version = None with quiet(): with settings(warn_only=True): git_version = env.safe_run_output( "git --version").strip().split()[-1] if git_version and LooseVersion(git_version) < LooseVersion("1.7"): _install_pkg(env, "git", brew_cmd, ipkgs) for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba env.safe_run("{brew_cmd} remove --force {dep}".format(**locals())) for dependency in ["htslib"]: if dependency in packages: if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"] or dependency not in ipkgs["current"]): _install_pkg_latest(env, dependency, [], brew_cmd, ipkgs) if "cpanminus" in packages: _install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs) _install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs) cpan.install_packages(env) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run( "test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)) if test_access.failed and env.safe_exists( "%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _get_current_pkgs(env, brew_cmd): out = {} with quiet(): which_out = env.safe_run_output("{brew_cmd} which".format(**locals())) for line in which_out.split("\n"): if line: pkg, version = line.rstrip().split() if pkg.endswith(":"): pkg = pkg[:-1] out[pkg] = version return out
def _safe_update(env, brew_cmd, formula_repos, cur_taps): """Revert any taps if we fail to update due to local changes. """ with quiet(): with settings(warn_only=True): out = env.safe_run("%s update" % brew_cmd) if out.failed: for repo in formula_repos: if repo in cur_taps: env.safe_run("%s untap %s" % (brew_cmd, repo)) env.safe_run("%s update" % brew_cmd)
def _conda_cmd(env): if hasattr(env, "conda_cmd") and env.conda_cmd: return env.conda_cmd to_check = [] if env.hosts == ["localhost"]: to_check.append(os.path.join(os.path.dirname(os.path.realpath(sys.executable)), "conda")) to_check.extend([os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda"]) for cmd in to_check: with quiet(): test = env.safe_run("%s --version" % cmd) if test.succeeded: return cmd return None
def _get_current_pkgs(env, brew_cmd): out = {} with quiet(): which_out = env.safe_run_output("{brew_cmd} list --versions".format(**locals())) for line in which_out.split("\n"): if line: parts = line.rstrip().split() if len(parts) == 2: pkg, version = line.rstrip().split() if pkg.endswith(":"): pkg = pkg[:-1] out[pkg] = version return out
def download(self, seq_dir): zipped_file = None genome_file = "%s.fa" % self._name if not self._exists(genome_file, seq_dir): prep_dir = "seq_prep" env.safe_run("mkdir -p %s" % prep_dir) with cd(prep_dir): zipped_file = self._download_zip(seq_dir) if zipped_file.endswith(".tar.gz"): env.safe_run("tar -xzpf %s" % zipped_file) elif zipped_file.endswith(".zip"): env.safe_run("unzip %s" % zipped_file) elif zipped_file.endswith(".gz"): if not env.safe_exists("out.fa"): env.safe_run("gunzip -c %s > out.fa" % zipped_file) else: raise ValueError("Do not know how to handle: %s" % zipped_file) tmp_file = genome_file.replace(".fa", ".txt") result = env.safe_run_output("find `pwd` -name '*.fa'") result = [x.strip() for x in result.split("\n")] if len(result) == 1: orig_result = result[0] result = self._split_multifasta(result[0]) env.safe_run("rm %s" % orig_result) result = self._karyotype_sort(result) env.safe_run("rm -f inputs.txt") for fname in result: with quiet(): env.safe_run("echo '%s' >> inputs.txt" % fname) env.safe_run("cat `cat inputs.txt` > %s" % (tmp_file)) for fname in result: with quiet(): env.safe_run("rm -f %s" % fname) env.safe_run("mv %s %s" % (tmp_file, genome_file)) zipped_file = os.path.join(prep_dir, zipped_file) genome_file = os.path.join(prep_dir, genome_file) return genome_file, [zipped_file]
def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. - Installation of required Perl libraries. - Upgrades any package dependencies """ for dep in ["openssl"]: _safe_link_pkg(env, dep, brew_cmd) for dep in ["expat", "pkg-config", "xz", "unzip"]: _install_pkg(env, dep, brew_cmd, ipkgs) # check if we have an older git and need to install it from brew git_version = None with quiet(): with settings(warn_only=True): git_version = env.safe_run_output("git --version").strip().split()[-1] if git_version and LooseVersion(git_version) < LooseVersion("1.7"): _install_pkg(env, "git", brew_cmd, ipkgs) for dep in ["sambamba"]: # Avoid conflict with homebrew-science sambamba env.safe_run("{brew_cmd} remove --force {dep}".format(**locals())) for dependency in ["htslib"]: if dependency in packages: if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"] or dependency not in ipkgs["current"]): _install_pkg_latest(env, dependency, [], brew_cmd, ipkgs) if "cpanminus" in packages: _install_pkg_latest(env, "cpanminus", [], brew_cmd, ipkgs) _install_pkg_latest(env, "samtools-library-0.1", [], brew_cmd, ipkgs) cpan.install_packages(env) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)) if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _custom_unlink(env, brew_cmd, pkg): """Handle custom unlinking of packages that can break builds of others. Does a temporary unlink and relink of packages while building. """ unlinks = {"lumpy-sv": ["bamtools"]} for upkg in unlinks.get(pkg, []): _safe_unlink_pkg(env, upkg, brew_cmd) try: yield None finally: for upkg in unlinks.get(pkg, []): with settings(warn_only=True): with quiet(): env.safe_run("%s link --overwrite %s" % (brew_cmd, upkg))
def _symlink_bin(package, env, conda_info, conda_bin, files=None, prefix=""): """Symlink files installed in the bin directory into install directory. """ package = package.split("=")[0] final_bindir = os.path.join(env.system_install, "bin") for pkg_subdir in json.loads(env.safe_run_output("{conda_bin} list --json -f {package}".format(**locals()))): for pkg_dir in conda_info["pkgs_dirs"]: pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin") if env.safe_exists(pkg_bindir): if not files: with quiet(): files = env.safe_run_output("ls -1 {pkg_bindir}".format(**locals())).split() for fname in files: _do_symlink(os.path.join(pkg_bindir, fname), os.path.join(final_bindir, "%s%s" % (prefix, fname)))
def _pip_cmd(env): """Retrieve pip command for installing python packages, allowing configuration. """ to_check = ["pip"] if "pip_cmd" in env and env.pip_cmd: to_check.append(env.pip_cmd) if not env.use_sudo: to_check.append(os.path.join(env.system_install, "bin", "pip")) if "python_version_ext" in env and env.python_version_ext: to_check.append("pip-{0}".format(env.python_version_ext)) for cmd in to_check: with quiet(): pip_version = env.safe_run("%s --version" % cmd) if pip_version.succeeded: return cmd raise ValueError("Could not find pip installer from: %s" % to_check)
def up_to_date(env, cmd, version, args=None, stdout_flag=None, stdout_index=-1): """Check if the given command is up to date with the provided version. """ if shared._executable_not_on_path(cmd): return False if args: cmd = cmd + " " + " ".join(args) with quiet(): path_safe = "export PATH=$PATH:%s/bin && " % env.system_install out = env.safe_run_output(path_safe + cmd) if stdout_flag: iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index) else: iversion = out.strip() iversion = _clean_version(iversion) return LooseVersion(iversion) >= LooseVersion(version)
def _safe_update(env, brew_cmd, formula_repos, cur_taps): """Revert any taps if we fail to update due to local changes. """ with _git_stash(env, brew_cmd): with quiet(): with settings(warn_only=True): out = env.safe_run("%s update" % brew_cmd) if out.failed: for repo in formula_repos: if repo in cur_taps: env.safe_run("%s untap %s" % (brew_cmd, repo)) with settings(warn_only=True): out = env.safe_run("%s update" % brew_cmd) if out.failed: print("\n\nHomebrew update failed.") print("You might need to upgrade git by installing inside bcbio with:") print("'brew install git --env=inherit --ignore-dependences'\n\n")
def _conda_cmd(env): if hasattr(env, "conda_cmd") and env.conda_cmd: return env.conda_cmd to_check = [] if env.hosts == ["localhost"]: to_check.append( os.path.join(os.path.dirname(os.path.realpath(sys.executable)), "conda")) to_check.extend([ os.path.join(env.system_install, "anaconda", "bin", "conda"), "conda" ]) for cmd in to_check: with quiet(): test = env.safe_run("%s --version" % cmd) if test.succeeded: return cmd return None
def _symlink_bin(package, env, conda_info, conda_bin, files=None, prefix=""): """Symlink files installed in the bin directory into install directory. """ package = package.split("=")[0] final_bindir = os.path.join(env.system_install, "bin") for pkg_subdir in json.loads( env.safe_run_output( "{conda_bin} list --json -f {package}".format(**locals()))): for pkg_dir in conda_info["pkgs_dirs"]: pkg_bindir = os.path.join(pkg_dir, pkg_subdir, "bin") if env.safe_exists(pkg_bindir): if not files: with quiet(): files = env.safe_run_output( "ls -1 {pkg_bindir}".format(**locals())).split() for fname in files: _do_symlink( os.path.join(pkg_bindir, fname), os.path.join(final_bindir, "%s%s" % (prefix, fname)))
def get_installed_version(env, cmd, version, args=None, stdout_flag=None, stdout_index=-1): """Check if the given command is up to date with the provided version. """ if shared._executable_not_on_path(cmd): return False if args: cmd = cmd + " " + " ".join(args) with quiet(): path_safe = ("export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:{s}/lib/pkgconfig && " "export PATH=$PATH:{s}/bin && " "export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:{s}/lib && ".format(s=env.system_install)) out = env.safe_run_output(path_safe + cmd) if stdout_flag: iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index) else: iversion = out.strip() iversion = _clean_version(iversion) return iversion
def _pip_cmd(env): """Retrieve pip command for installing python packages, allowing configuration. """ anaconda_pip = os.path.join(env.system_install, "anaconda", "bin", "pip") if env.safe_exists(anaconda_pip): to_check = [anaconda_pip] else: to_check = ["pip"] if "pip_cmd" in env and env.pip_cmd: to_check.append(env.pip_cmd) if not env.use_sudo: to_check.append(os.path.join(env.system_install, "bin", "pip")) if "python_version_ext" in env and env.python_version_ext: to_check.append("pip-{0}".format(env.python_version_ext)) for cmd in to_check: with quiet(): pip_version = env.safe_run("%s --version" % cmd) if pip_version.succeeded: return cmd raise ValueError("Could not find pip installer from: %s" % to_check)
def up_to_date(env, cmd, version, args=None, stdout_flag=None, stdout_index=-1): """Check if the given command is up to date with the provided version. """ if shared._executable_not_on_path(cmd): return False if args: cmd = cmd + " " + " ".join(args) with quiet(): path_safe = "export PATH=$PATH:%s/bin && " out = env.safe_run_output(path_safe + cmd) if stdout_flag: iversion = _parse_from_stdoutflag(out, stdout_flag, stdout_index) else: iversion = out.strip() iversion = _clean_version(iversion) return LooseVersion(iversion) >= LooseVersion(version)
def _determine_distribution(env): """ Attempt to automatically determine the distribution of the target machine. Currently works for Ubuntu, CentOS, Debian, Scientific Linux and Mac OS X. """ with quiet(): output = env.safe_run_output("cat /etc/*release").lower() if output.find("id=ubuntu") >= 0: return "ubuntu" elif output.find("centos release") >= 0: return "centos" elif output.find("centos linux release") >= 0: return "centos" elif output.find("red hat enterprise linux") >= 0: return "centos" elif output.find("fedora") >= 0: return "centos" # Amazon AMIs are Red-Hat based elif output.find("amzn") >= 0 or output.find("amazon") >= 0: return "centos" elif output.find("suse linux") >= 0: return "suse" elif output.find("opensuse") >= 0: return "suse" elif output.find("scientific linux") >= 0: return "scientificlinux" elif env.safe_exists("/etc/debian_version"): return "debian" elif output.find("id=arch") >= 0 or output.find('id_like="arch"') >= 0: return "arch" elif output.find("antergos") >= 0: return "arch" # check for file used by Python's platform.mac_ver elif env.safe_exists("/System/Library/CoreServices/SystemVersion.plist"): return "macosx" else: raise Exception( "Attempt to automatically determine Linux distribution of target machine failed:\n%s" % output)
def _install_brew_baseline(env, brew_cmd, ipkgs, packages): """Install baseline brew components not handled by dependency system. - Installation of required Perl libraries. - Ensures installed samtools does not overlap with bcftools - Upgrades any package dependencies """ for dep in ["cpanminus", "expat"]: _install_pkg_latest(env, dep, brew_cmd, ipkgs) # if installing samtools, avoid bcftools conflicts if len([x for x in packages if x.find("samtools") >= 0]): with settings(warn_only=True): try: has_bcftools = int(env.safe_run_output("{brew_cmd} list samtools | grep -c bcftools".format( brew_cmd=brew_cmd))) except ValueError: has_bcftools = 0 if has_bcftools: env.safe_run("{brew_cmd} uninstall {pkg}".format(brew_cmd=brew_cmd, pkg="samtools")) ipkgs["current"].pop("samtools", None) _install_pkg_latest(env, "samtools", brew_cmd, ipkgs, "--without-bcftools") for dependency in ["htslib", "libmaus"]: if (dependency in ipkgs["outdated"] or "chapmanb/cbl/%s" % dependency in ipkgs["outdated"] or dependency not in ipkgs["current"]): _install_pkg_latest(env, dependency, brew_cmd, ipkgs) cpanm_cmd = os.path.join(os.path.dirname(brew_cmd), "cpanm") for perl_lib in ["Statistics::Descriptive"]: env.safe_run("%s -i --notest --local-lib=%s '%s'" % (cpanm_cmd, env.system_install, perl_lib)) # Ensure paths we may have missed on install are accessible to regular user if env.use_sudo: paths = ["share", "share/java"] for path in paths: with quiet(): test_access = env.safe_run("test -d %s/%s && test -O %s/%s" % (env.system_install, path, env.system_install, path)) if test_access.failed and env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo("chown %s %s/%s" % (env.user, env.system_install, path))
def _safe_uninstall_pkg(env, pkg_str, brew_cmd): """Uninstall packages which get pulled in even when unlinked by brew. """ with settings(warn_only=True): with quiet(): env.safe_run("{brew_cmd} uninstall {pkg_str}".format(**locals()))
def install_homebrew(env): """Homebrew package manager for OSX and Linuxbrew for linux systems. https://github.com/mxcl/homebrew https://github.com/Homebrew/linuxbrew """ if env.distribution == "macosx": with quiet(): test_brewcmd = env.safe_run("brew --version") if not test_brewcmd.succeeded: env.safe_run( 'ruby -e "$(curl -fsSL https://raw.github.com/mxcl/homebrew/go/install)"' ) else: brewcmd = os.path.join(env.system_install, "bin", "brew") with quiet(): test_brewcmd = env.safe_run("%s --version" % brewcmd) if not test_brewcmd.succeeded or _linuxbrew_origin_problem(brewcmd): with shared._make_tmp_dir() as tmp_dir: with cd(tmp_dir): if env.safe_exists("linuxbrew"): env.safe_run("rm -rf linuxbrew") for cleandir in ["Library", ".git"]: if env.safe_exists("%s/%s" % (env.system_install, cleandir)): env.safe_run("rm -rf %s/%s" % (env.system_install, cleandir)) env.safe_run( "git clone https://github.com/Homebrew/linuxbrew.git") with cd("linuxbrew"): if not env.safe_exists(env.system_install): env.safe_sudo("mkdir -p %s" % env.system_install) env.safe_sudo("chown %s %s" % (env.user, env.system_install)) paths = [ "bin", "etc", "include", "lib", "lib/pkgconfig", "Library", "sbin", "share", "var", "var/log", "share/java", "share/locale", "share/man", "share/man/man1", "share/man/man2", "share/man/man3", "share/man/man4", "share/man/man5", "share/man/man6", "share/man/man7", "share/man/man8", "share/info", "share/doc", "share/aclocal", "lib/python2.7/site-packages", "lib/python2.6/site-packages", "lib/python3.2/site-packages", "lib/python3.3/site-packages", "lib/perl5", "lib/perl5/site_perl" ] if not env.safe_exists("%s/bin" % env.system_install): env.safe_sudo("mkdir -p %s/bin" % env.system_install) for path in paths: if env.safe_exists("%s/%s" % (env.system_install, path)): env.safe_sudo( "chown %s %s/%s" % (env.user, env.system_install, path)) if not env.safe_exists( "%s/Library" % env.system_install): env.safe_run("mv Library %s" % env.system_install) if not env.safe_exists("%s/.git" % env.system_install): env.safe_run("mv .git %s" % env.system_install) man_dir = "share/man/man1" if not env.safe_exists("%s/%s" % (env.system_install, man_dir)): env.safe_run("mkdir -p %s/%s" % (env.system_install, man_dir)) env.safe_run("mv -f %s/brew.1 %s/%s" % (man_dir, env.system_install, man_dir)) env.safe_run("mv -f bin/brew %s/bin" % env.system_install)
def _safe_link_pkg(env, pkg_str, brew_cmd): """Link packages required for builds, but not necessarily installed """ with settings(warn_only=True): with quiet(): env.safe_run("{brew_cmd} link --overwrite {pkg_str}".format(**locals()))
def _safe_unlink_pkg(env, pkg_str, brew_cmd): """Unlink packages which can cause issues with a Linux system. """ with settings(warn_only=True): with quiet(): env.safe_run("{brew_cmd} unlink {pkg_str}".format(**locals()))