def _setup_env(self):
"""
Setup a custom user-level env
"""
# Add commond directories to PATH
path_additions = ("export PATH=/usr/lib/postgresql/9.1/bin:" +
"/usr/nginx/sbin:/mnt/galaxy/tools/bin:$PATH")
env.logger.debug("Amending the PATH with {0}".format(path_additions))
_add_to_profiles(path_additions, ['/etc/bash.bashrc'])
# Seed the history with frequently used commands
env.logger.debug("Setting bash history")
local = os.path.join(env.config_dir, os.pardir, "installed_files",
"bash_history")
remote = os.path.join('/home', 'ubuntu', '.bash_history')
put(local, remote, mode=0660, use_sudo=True)
# Install ipython profiles
users = ['ubuntu', 'galaxy']
for user in users:
env.logger.debug(
"Setting installing ipython profile for user {0}".format(user))
env.safe_sudo("su - {0} -c 'ipython profile create'".format(user))
local = os.path.join(env.config_dir, os.pardir, "installed_files",
"ipython_config.py")
remote = os.path.join('/home', user, '.ipython', 'profile_default',
"ipython_config.py")
put(local, remote, mode=0644, use_sudo=True)
env.safe_sudo("chown {0}:{0} {1}".format(user, remote))
def _setup_env(self):
"""
Setup a custom user-level env
"""
# Add commond directories to PATH
path_additions = ("export PATH=/usr/lib/postgresql/9.1/bin:" +
"/usr/nginx/sbin:/mnt/galaxy/tools/bin:$PATH")
env.logger.debug("Amending the PATH with {0}".format(path_additions))
_add_to_profiles(path_additions, ['/etc/bash.bashrc'])
# Seed the history with frequently used commands
env.logger.debug("Setting bash history")
local = os.path.join(env.config_dir, os.pardir, "installed_files",
"bash_history")
remote = os.path.join('/home', 'ubuntu', '.bash_history')
put(local, remote, mode=0660, use_sudo=True)
# Install ipython profiles
users = ['ubuntu', 'galaxy']
for user in users:
env.logger.debug("Setting installing ipython profile for user {0}"
.format(user))
env.safe_sudo("su - {0} -c 'ipython profile create'".format(user))
local = os.path.join(env.config_dir, os.pardir, "installed_files",
"ipython_config.py")
remote = os.path.join('/home', user, '.ipython', 'profile_default',
"ipython_config.py")
put(local, remote, mode=0644, use_sudo=True)
env.safe_sudo("chown {0}:{0} {1}".format(user, remote))
def r_library_installer(config):
"""Install R libraries using CRAN and Bioconductor.
"""
# Create an Rscript file with install details.
out_file = "install_packages.R"
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
env.safe_run("touch %s" % out_file)
lib_loc = os.path.join(env.system_install, "lib", "R", "site-library")
env.safe_sudo("mkdir -p %s" % lib_loc)
repo_info = """
.libPaths(c("%s"))
library(methods)
cran.repos <- getOption("repos")
cran.repos["CRAN" ] <- "%s"
options(repos=cran.repos)
source("%s")
""" % (lib_loc, config["cranrepo"], config["biocrepo"])
env.safe_append(out_file, repo_info)
install_fn = """
repo.installer <- function(repos, install.fn) {
%s
maybe.install <- function(pname) {
if (!(pname %%in%% installed.packages()))
install.fn(pname)
}
}
"""
if config.get("update_packages", True):
update_str = """
update.packages(lib.loc="%s", repos=repos, ask=FALSE)
""" % lib_loc
else:
update_str = "\n"
env.safe_append(out_file, install_fn % update_str)
std_install = """
std.pkgs <- c(%s)
std.installer = repo.installer(cran.repos, install.packages)
lapply(std.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran']))
env.safe_append(out_file, std_install)
if len(config.get("bioc", [])) > 0:
bioc_install = """
bioc.pkgs <- c(%s)
bioc.installer = repo.installer(biocinstallRepos(), biocLite)
lapply(bioc.pkgs, bioc.installer)
""" % (", ".join('"%s"' % p for p in config['bioc']))
env.safe_append(out_file, bioc_install)
# run the script and then get rid of it
rscript = fabutils.find_cmd(env, "Rscript", "--version")
if rscript:
env.safe_sudo("%s %s" % (rscript, out_file))
else:
env.logger.warn("Rscript not found; skipping install of R libraries.")
env.safe_run("rm -f %s" % out_file)
def r_library_installer(config):
"""Install R libraries using CRAN and Bioconductor.
"""
# Create an Rscript file with install details.
out_file = "install_packages.R"
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
env.safe_run("touch %s" % out_file)
lib_loc = os.path.join(env.system_install, "lib", "R", "site-library")
env.safe_sudo("mkdir -p %s" % lib_loc)
repo_info = """
.libPaths(c("%s"))
library(methods)
cran.repos <- getOption("repos")
cran.repos["CRAN" ] <- "%s"
options(repos=cran.repos)
source("%s")
""" % (lib_loc, config["cranrepo"], config["biocrepo"])
env.safe_append(out_file, repo_info)
install_fn = """
repo.installer <- function(repos, install.fn) {
%s
maybe.install <- function(pname) {
if (!(pname %%in%% installed.packages()))
install.fn(pname)
}
}
"""
if config.get("update_packages", True):
update_str = """
update.packages(lib.loc="%s", repos=repos, ask=FALSE)
""" % lib_loc
else:
update_str = "\n"
env.safe_append(out_file, install_fn % update_str)
std_install = """
std.pkgs <- c(%s)
std.installer = repo.installer(cran.repos, install.packages)
lapply(std.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran']))
env.safe_append(out_file, std_install)
if len(config.get("bioc", [])) > 0:
bioc_install = """
bioc.pkgs <- c(%s)
bioc.installer = repo.installer(biocinstallRepos(), biocLite)
lapply(bioc.pkgs, bioc.installer)
""" % (", ".join('"%s"' % p for p in config['bioc']))
env.safe_append(out_file, bioc_install)
# run the script and then get rid of it
rscript = fabutils.find_cmd(env, "Rscript", "--version")
if rscript:
env.safe_sudo("%s %s" % (rscript, out_file))
else:
env.logger.warn("Rscript not found; skipping install of R libraries.")
env.safe_run("rm -f %s" % out_file)
def detach_volumes(vm_launcher, options):
volumes = options.get("volumes", [])
if not volumes:
return
boto_connection = vm_launcher.boto_connection()
instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id")
for volume in volumes:
volume_id = volume['id']
path = volume.get("path")
env.safe_sudo("umount '%s'" % path)
_detach(boto_connection, instance_id, volume_id)
def _make_snapshot(vm_launcher, fs_path, desc):
""" Create a snapshot of an existing volume that is currently attached to an
instance, taking care of the unmounting and detaching. If you specify the
optional argument (:galaxy), the script will pull the latest Galaxy code
from bitbucket and perform an update before snapshotting. Else, the script
will prompt for the file system path to be snapshoted.
In order for this to work, an instance on EC2 needs to be running with a
volume that wants to be snapshoted attached and mounted. The script will
unmount the volume, create a snaphost and offer to reattach and mount the
volume or create a new one from the freshly created snapshot.
Except for potentially Galaxy, MAKE SURE there are no running processes
using the volume and that no one is logged into the instance and sitting
in the given directory.
"""
instance_id = run(
"curl --silent http://169.254.169.254/latest/meta-data/instance-id")
availability_zone = run(
"curl --silent http://169.254.169.254/latest/meta-data/placement/availability-zone"
)
instance_region = availability_zone[:
-1] # Truncate zone letter to get region name
# Find the device where the file system is mounted to
# Find the EBS volume where the file system resides
device_id = _find_mounted_device_id(fs_path)
ec2_conn = vm_launcher.boto_connection()
fs_vol = _get_attached(ec2_conn, instance_id, device_id)
if fs_vol:
env.safe_sudo("umount %s" % fs_path)
_detach(ec2_conn, instance_id, fs_vol.id)
snap_id = _create_snapshot(ec2_conn, fs_vol.id, desc)
# TODO: Auto Update snaps?
make_public = True
if make_public: # Make option
ec2_conn.modify_snapshot_attribute(
snap_id,
attribute='createVolumePermission',
operation='add',
groups=['all'])
reattach = True
if reattach:
_attach(ec2_conn, instance_id, fs_vol.id, device_id)
env.safe_sudo("mount %s %s" % (device_id, fs_path))
delete_old_volume = False
if delete_old_volume:
_delete_volume(ec2_conn, fs_vol.id)
print("----- Done snapshoting volume '%s' for file system '%s' -----" %
(fs_vol.id, fs_path))
else:
print(
"ERROR: Failed to find require file system, is boto installed? Is it not actually mounted?"
)
def detach_volumes(vm_launcher, options):
volumes = options.get("volumes", [])
if not volumes:
return
boto_connection = vm_launcher.boto_connection()
instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id")
for volume in volumes:
volume_id = volume['id']
path = volume.get("path")
env.safe_sudo("umount '%s'" % path)
_detach(boto_connection, instance_id, volume_id)
def _make_install_script(out_file, config):
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
env.safe_run("touch %s" % out_file)
lib_loc = os.path.join(env.system_install, "lib", "R", "site-library")
env.safe_sudo("mkdir -p %s" % lib_loc)
with settings(warn_only=True):
env.safe_sudo("chown -R %s %s" % (env.user, lib_loc))
repo_info = """
.libPaths(c("%s"))
library(methods)
cran.repos <- getOption("repos")
cran.repos["CRAN" ] <- "%s"
options(repos=cran.repos)
source("%s")
""" % (lib_loc, config["cranrepo"], config["biocrepo"])
env.safe_append(out_file, repo_info)
install_fn = """
repo.installer <- function(repos, install.fn) {
%s
maybe.install <- function(pname) {
if (!(pname %%in%% installed.packages()))
install.fn(pname)
}
}
"""
if config.get("update_packages", True):
update_str = """
update.packages(lib.loc="%s", repos=repos, ask=FALSE)
""" % lib_loc
else:
update_str = "\n"
env.safe_append(out_file, install_fn % update_str)
std_install = """
std.pkgs <- c(%s)
std.installer = repo.installer(cran.repos, install.packages)
lapply(std.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran']))
env.safe_append(out_file, std_install)
if len(config.get("bioc", [])) > 0:
bioc_install = """
bioc.pkgs <- c(%s)
bioc.installer = repo.installer(biocinstallRepos(), biocLite)
lapply(bioc.pkgs, bioc.installer)
""" % (", ".join('"%s"' % p for p in config['bioc']))
env.safe_append(out_file, bioc_install)
if config.get("cran-after-bioc"):
std2_install = """
std2.pkgs <- c(%s)
lapply(std2.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran-after-bioc']))
env.safe_append(out_file, std2_install)
def attach_volumes(vm_launcher, options, format=False):
"""
"""
volumes = options.get("volumes", [])
if not volumes:
return
boto_connection = vm_launcher.boto_connection()
instance_id = run(
"curl --silent http://169.254.169.254/latest/meta-data/instance-id")
for volume in volumes:
volume_id = volume['id']
device_id = volume['device']
if not _get_attached(boto_connection,
instance_id,
device_id,
valid_states=["attached", "attaching"]):
boto_connection.attach_volume(volume_id, instance_id, device_id)
for volume in volumes:
volume_id = volume['id']
device_id = volume['device']
path = volume.get("path")
while True:
if _get_attached(boto_connection, instance_id, device_id):
break
sleep(5)
print("Waiting for volume corresponding to device %s to attach" %
device_id)
break
# Don't mount if already mounted
if _find_mounted_device_id(path):
continue
format = str(volume.get('format', "False")).lower()
if format == "true":
_format_device(device_id)
env.safe_sudo("mkdir -p '%s'" % path)
try:
_mount(device_id, path)
except:
if format == "__auto__":
print(
"Failed to mount device. format is set to __auto__ so will now format device and retry mount"
)
_format_device(device_id)
_mount(device_id, path)
else:
raise
def _make_install_script(out_file, config):
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
env.safe_run("touch %s" % out_file)
lib_loc = os.path.join(env.system_install, "lib", "R", "site-library")
env.safe_sudo("mkdir -p %s" % lib_loc)
repo_info = """
.libPaths(c("%s"))
library(methods)
cran.repos <- getOption("repos")
cran.repos["CRAN" ] <- "%s"
options(repos=cran.repos)
source("%s")
""" % (lib_loc, config["cranrepo"], config["biocrepo"])
env.safe_append(out_file, repo_info)
install_fn = """
repo.installer <- function(repos, install.fn) {
%s
maybe.install <- function(pname) {
if (!(pname %%in%% installed.packages()))
install.fn(pname)
}
}
"""
if config.get("update_packages", True):
update_str = """
update.packages(lib.loc="%s", repos=repos, ask=FALSE)
""" % lib_loc
else:
update_str = "\n"
env.safe_append(out_file, install_fn % update_str)
std_install = """
std.pkgs <- c(%s)
std.installer = repo.installer(cran.repos, install.packages)
lapply(std.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran']))
env.safe_append(out_file, std_install)
if len(config.get("bioc", [])) > 0:
bioc_install = """
bioc.pkgs <- c(%s)
bioc.installer = repo.installer(biocinstallRepos(), biocLite)
lapply(bioc.pkgs, bioc.installer)
""" % (", ".join('"%s"' % p for p in config['bioc']))
env.safe_append(out_file, bioc_install)
if config.get("cran-after-bioc"):
std2_install = """
std2.pkgs <- c(%s)
lapply(std2.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran-after-bioc']))
env.safe_append(out_file, std2_install)
def r_library_installer(config):
"""Install R libraries using CRAN and Bioconductor.
"""
with shared._make_tmp_dir() as tmp_dir:
with cd(tmp_dir):
# Create an Rscript file with install details.
out_file = os.path.join(tmp_dir, "install_packages.R")
_make_install_script(out_file, config)
# run the script and then get rid of it
rscript = fabutils.find_cmd(env, "Rscript", "--version")
if rscript:
env.safe_sudo("%s %s" % (rscript, out_file))
else:
env.logger.warn("Rscript not found; skipping install of R libraries.")
env.safe_run("rm -f %s" % out_file)
def r_library_installer(config):
"""Install R libraries using CRAN and Bioconductor.
"""
# Create an Rscript file with install details.
out_file = "install_packages.R"
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
env.safe_run("touch %s" % out_file)
repo_info = """
cran.repos <- getOption("repos")
cran.repos["CRAN" ] <- "%s"
options(repos=cran.repos)
source("%s")
""" % (config["cranrepo"], config["biocrepo"])
env.safe_append(out_file, repo_info)
install_fn = """
repo.installer <- function(repos, install.fn) {
update.or.install <- function(pname) {
if (pname %in% installed.packages())
update.packages(lib.loc=c(pname), repos=repos, ask=FALSE)
else
install.fn(pname)
}
}
"""
env.safe_append(out_file, install_fn)
std_install = """
std.pkgs <- c(%s)
std.installer = repo.installer(cran.repos, install.packages)
lapply(std.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran']))
env.safe_append(out_file, std_install)
if len(config.get("bioc", [])) > 0:
bioc_install = """
bioc.pkgs <- c(%s)
bioc.installer = repo.installer(biocinstallRepos(), biocLite)
lapply(bioc.pkgs, bioc.installer)
""" % (", ".join('"%s"' % p for p in config['bioc']))
env.safe_append(out_file, bioc_install)
if config.get("update_packages", True):
final_update = """
update.packages(repos=biocinstallRepos(), ask=FALSE)
update.packages(ask=FALSE)
"""
env.safe_append(out_file, final_update)
# run the script and then get rid of it
env.safe_sudo("Rscript %s" % out_file)
env.safe_run("rm -f %s" % out_file)
def r_library_installer(config):
"""Install R libraries using CRAN and Bioconductor.
"""
with shared._make_tmp_dir() as tmp_dir:
with cd(tmp_dir):
# Create an Rscript file with install details.
out_file = os.path.join(tmp_dir, "install_packages.R")
_make_install_script(out_file, config)
# run the script and then get rid of it
rscript = fabutils.find_cmd(env, "Rscript", "--version")
if rscript:
env.safe_sudo("%s %s" % (rscript, out_file))
else:
env.logger.warn(
"Rscript not found; skipping install of R libraries.")
env.safe_run("rm -f %s" % out_file)
def setup_taxonomy_data():
taxonomy_directory = os.path.join(env.data_files, "taxonomy")
env.safe_sudo("mkdir -p '%s'" % taxonomy_directory, user=env.user)
with cd(taxonomy_directory):
taxonomy_url = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz"
gi_taxid_nucl = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/gi_taxid_nucl.dmp.gz"
gi_taxid_prot = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/gi_taxid_prot.dmp.gz"
wget(taxonomy_url)
wget(gi_taxid_nucl)
wget(gi_taxid_prot)
run("gunzip -c taxdump.tar.gz | tar xvf -")
run("gunzip gi_taxid_nucl.dmp.gz")
run("gunzip gi_taxid_prot.dmp.gz")
run("cat gi_taxid_nucl.dmp gi_taxid_prot.dmp > gi_taxid_all.dmp")
run("sort -n -k 1 gi_taxid_all.dmp > gi_taxid_sorted.txt")
run("rm gi_taxid_nucl.dmp gi_taxid_prot.dmp gi_taxid_all.dmp")
run("cat names.dmp | sed s/[\\(\\)\\'\\\"]/_/g > names.temporary")
run("mv names.dmp names.dmp.orig")
run("mv names.temporary names.dmp")
def setup_taxonomy_data():
taxonomy_directory = os.path.join(env.data_files, "taxonomy")
env.safe_sudo("mkdir -p '%s'" % taxonomy_directory, user=env.user)
with cd(taxonomy_directory):
taxonomy_url = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz"
gi_taxid_nucl = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/gi_taxid_nucl.dmp.gz"
gi_taxid_prot = "ftp://ftp.ncbi.nih.gov/pub/taxonomy/gi_taxid_prot.dmp.gz"
wget(taxonomy_url)
wget(gi_taxid_nucl)
wget(gi_taxid_prot)
run("gunzip -c taxdump.tar.gz | tar xvf -")
run("gunzip gi_taxid_nucl.dmp.gz")
run("gunzip gi_taxid_prot.dmp.gz")
run("cat gi_taxid_nucl.dmp gi_taxid_prot.dmp > gi_taxid_all.dmp")
run("sort -n -k 1 gi_taxid_all.dmp > gi_taxid_sorted.txt")
run("rm gi_taxid_nucl.dmp gi_taxid_prot.dmp gi_taxid_all.dmp")
run("cat names.dmp | sed s/[\\(\\)\\'\\\"]/_/g > names.temporary")
run("mv names.dmp names.dmp.orig")
run("mv names.temporary names.dmp")
def attach_volumes(vm_launcher, options, format=False):
"""
"""
volumes = options.get("volumes", [])
if not volumes:
return
boto_connection = vm_launcher.boto_connection()
instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id")
for volume in volumes:
volume_id = volume['id']
device_id = volume['device']
if not _get_attached(boto_connection, instance_id, device_id, valid_states=["attached", "attaching"]):
boto_connection.attach_volume(volume_id, instance_id, device_id)
for volume in volumes:
volume_id = volume['id']
device_id = volume['device']
path = volume.get("path")
while True:
if _get_attached(boto_connection, instance_id, device_id):
break
sleep(5)
print("Waiting for volume corresponding to device %s to attach" % device_id)
break
# Don't mount if already mounted
if _find_mounted_device_id(path):
continue
format = str(volume.get('format', "False")).lower()
if format == "true":
_format_device(device_id)
env.safe_sudo("mkdir -p '%s'" % path)
try:
_mount(device_id, path)
except:
if format == "__auto__":
print("Failed to mount device. format is set to __auto__ so will now format device and retry mount")
_format_device(device_id)
_mount(device_id, path)
else:
raise
def _make_snapshot(vm_launcher, fs_path, desc):
""" Create a snapshot of an existing volume that is currently attached to an
instance, taking care of the unmounting and detaching. If you specify the
optional argument (:galaxy), the script will pull the latest Galaxy code
from bitbucket and perform an update before snapshotting. Else, the script
will prompt for the file system path to be snapshoted.
In order for this to work, an instance on EC2 needs to be running with a
volume that wants to be snapshoted attached and mounted. The script will
unmount the volume, create a snaphost and offer to reattach and mount the
volume or create a new one from the freshly created snapshot.
Except for potentially Galaxy, MAKE SURE there are no running processes
using the volume and that no one is logged into the instance and sitting
in the given directory.
"""
instance_id = run("curl --silent http://169.254.169.254/latest/meta-data/instance-id")
availability_zone = run("curl --silent http://169.254.169.254/latest/meta-data/placement/availability-zone")
instance_region = availability_zone[:-1] # Truncate zone letter to get region name
# Find the device where the file system is mounted to
# Find the EBS volume where the file system resides
device_id = _find_mounted_device_id(fs_path)
ec2_conn = vm_launcher.boto_connection()
fs_vol = _get_attached(ec2_conn, instance_id, device_id)
if fs_vol:
env.safe_sudo("umount %s" % fs_path)
_detach(ec2_conn, instance_id, fs_vol.id)
snap_id = _create_snapshot(ec2_conn, fs_vol.id, desc)
# TODO: Auto Update snaps?
make_public = True
if make_public: # Make option
ec2_conn.modify_snapshot_attribute(snap_id, attribute='createVolumePermission', operation='add', groups=['all'])
reattach = True
if reattach:
_attach(ec2_conn, instance_id, fs_vol.id, device_id)
env.safe_sudo("mount %s %s" % (device_id, fs_path))
delete_old_volume = False
if delete_old_volume:
_delete_volume(ec2_conn, fs_vol.id)
print("----- Done snapshoting volume '%s' for file system '%s' -----" % (fs_vol.id, fs_path))
else:
print("ERROR: Failed to find require file system, is boto installed? Is it not actually mounted?")
def _install_modules_configure_make(self, env):
"""
Differences from standard _configure_make():
- TODO: ./configure with destination modulefile directory on shared filesystem
- add modules to profile for all users
"""
# currently putting module files in directory structure under env.system_install
# it would be better to store them on a filesystem shared with worker nodes; this is harder
env.safe_run("export PKG_CONFIG_PATH=$PKG_CONFIG_PATH:%s/lib/pkgconfig && " \
"./configure --prefix=%s " %
(env.system_install, env.system_install))
run('make')
env.safe_sudo('make install')
env.safe_sudo("cp etc/global/profile.modules /etc/profile.d/modules.sh")
env.safe_sudo("ln -s {0} {1}".format(os.path.join(env.system_install, 'Modules', env.environment_modules_version), os.path.join(env.system_install, 'Modules', 'default')))
def _make_install_script(out_file, config):
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
env.safe_run("touch %s" % out_file)
lib_loc = os.path.join(env.system_install, "lib", "R", "site-library")
env.safe_sudo("mkdir -p %s" % lib_loc)
with settings(warn_only=True):
env.safe_sudo("chown -R %s %s" % (env.user, lib_loc))
repo_info = """
.libPaths(c("%s"))
library(methods)
cran.repos <- getOption("repos")
cran.repos["CRAN" ] <- "%s"
options(repos=cran.repos)
source("%s")
""" % (lib_loc, config["cranrepo"], config["biocrepo"])
env.safe_append(out_file, repo_info)
install_fn = """
repo.installer <- function(repos, install.fn, pkg_name_fn) {
%s
maybe.install <- function(pname) {
check_name <- ifelse(is.null(pkg_name_fn), pname, pkg_name_fn(pname))
if (!(is.element(check_name, installed.packages()[,1])))
install.fn(pname)
}
}
"""
if config.get("update_packages", True):
update_str = """
update.packages(lib.loc="%s", repos=repos, ask=FALSE)
""" % lib_loc
else:
update_str = "\n"
env.safe_append(out_file, install_fn % update_str)
std_install = """
std.pkgs <- c(%s)
std.installer = repo.installer(cran.repos, install.packages, NULL)
lapply(std.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran']))
env.safe_append(out_file, std_install)
if len(config.get("bioc", [])) > 0:
bioc_install = """
bioc.pkgs <- c(%s)
bioc.installer = repo.installer(biocinstallRepos(), biocLite, NULL)
lapply(bioc.pkgs, bioc.installer)
""" % (", ".join('"%s"' % p for p in config['bioc']))
env.safe_append(out_file, bioc_install)
if config.get("cran-after-bioc"):
std2_install = """
std2.pkgs <- c(%s)
lapply(std2.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran-after-bioc']))
env.safe_append(out_file, std2_install)
if config.get("github"):
dev_install = """
library(devtools)
github.pkgs <- c(%s)
get_pkg_name <- function(orig) {
unlist(strsplit(unlist(strsplit(orig, "/"))[2], "@"))[1]
}
github_installer = repo.installer(NULL, install_github, get_pkg_name)
lapply(github.pkgs, github_installer)
""" % (", ".join('"%s"' % p for p in config['github']))
env.safe_append(out_file, dev_install)
def _start_nginx(env):
galaxy_data = env.galaxy_data_mount
env.safe_sudo("mkdir -p '%s'" % env.galaxy_data)
_chown_galaxy(env, galaxy_data)
start_service("nginx")
def _find_mounted_device_id(path):
# Adding dollar sign to grep to distinguish between /mnt/galaxy and /mnt/galaxyIndices
device_id = env.safe_sudo("df | grep '%s$' | awk '{print $1}'" % path)
return device_id
def purge_tools():
env.safe_sudo("rm -rf %s" % env.install_dir)
def _mount(device_id, path):
env.safe_sudo("mount '%s' '%s'" % (device_id, path))
def _format_device(device_id):
env.safe_sudo("mkfs -t ext3 %s" % device_id)
def _format_device(device_id):
env.safe_sudo("mkfs -t ext3 %s" % device_id)
def _mount(device_id, path):
env.safe_sudo("mount '%s' '%s'" % (device_id, path))
def _find_mounted_device_id(path):
# Adding dollar sign to grep to distinguish between /mnt/galaxy and /mnt/galaxyIndices
device_id = env.safe_sudo("df | grep '%s$' | awk '{print $1}'" % path)
return device_id
def _make_install_script(out_file, config):
if env.safe_exists(out_file):
env.safe_run("rm -f %s" % out_file)
env.safe_run("touch %s" % out_file)
lib_loc = os.path.join(env.system_install, "lib", "R", "site-library")
env.safe_sudo("mkdir -p %s" % lib_loc)
with settings(warn_only=True):
env.safe_sudo("chown -R %s %s" % (env.user, lib_loc))
repo_info = """
.libPaths(c("%s"))
library(methods)
cran.repos <- getOption("repos")
cran.repos["CRAN" ] <- "%s"
options(repos=cran.repos)
""" % (lib_loc, config["cranrepo"])
if config.get("biocrepo"):
repo_info += """\nsource("%s")\n""" % config["biocrepo"]
env.safe_append(out_file, repo_info)
install_fn = """
repo.installer <- function(repos, install.fn, pkg_name_fn) {
%s
maybe.install <- function(pname) {
if (!is.null(pkg_name_fn)) {
pinfo <- pkg_name_fn(pname)
ipkgs <- installed.packages()[,3][pinfo["pkg"]]
if (is.na(ipkgs[pinfo["pkg"]]) || pinfo["version"] != ipkgs[pinfo["pkg"]])
try(install.fn(pinfo["pname"]))
}
else if (!(is.element(pname, installed.packages()[,1])))
install.fn(pname)
}
}
"""
if config.get("update_packages", True):
update_str = """
update.packages(lib.loc="%s", repos=repos, ask=FALSE)
""" % lib_loc
else:
update_str = "\n"
env.safe_append(out_file, install_fn % update_str)
if len(config.get("cran") or []) > 0:
std_install = """
std.pkgs <- c(%s)
std.installer = repo.installer(cran.repos, install.packages, NULL)
lapply(std.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran']))
env.safe_append(out_file, std_install)
if len(config.get("bioc") or []) > 0:
bioc_install = """
bioc.pkgs <- c(%s)
bioc.installer = repo.installer(biocinstallRepos(), biocLite, NULL)
lapply(bioc.pkgs, bioc.installer)
""" % (", ".join('"%s"' % p for p in config['bioc']))
env.safe_append(out_file, bioc_install)
if config.get("cran-after-bioc"):
std2_install = """
std2.pkgs <- c(%s)
lapply(std2.pkgs, std.installer)
""" % (", ".join('"%s"' % p for p in config['cran-after-bioc']))
env.safe_append(out_file, std2_install)
if config.get("github"):
dev_install = """
library(devtools)
github.pkgs <- c(%s)
get_pkg_name <- function(orig) {
c(pkg=unlist(strsplit(unlist(strsplit(orig, "/"))[2], "@"))[1],
version=unlist(strsplit(orig, ";"))[2],
pname=unlist(strsplit(orig, ";"))[1])
}
gh_install <- function(name) {
install_github(name, upgrade_dependencies=FALSE)
}
github_installer = repo.installer(NULL, gh_install, get_pkg_name)
lapply(github.pkgs, github_installer)
""" % (", ".join('"%s"' % p for p in config['github']))
env.safe_append(out_file, dev_install)
def _start_nginx(env):
galaxy_data = env.galaxy_data_mount
env.safe_sudo("mkdir -p '%s'" % env.galaxy_data)
_chown_galaxy(env, galaxy_data)
start_service("nginx")
def _install_postgres_configure_make(self, env):
run('./configure --prefix=/usr/lib/postgresql/8.4 --with-pgport=5840 --with-python')
run('make')
env.safe_sudo('make install')
def _install_postgres_configure_make(self, env):
run('./configure --prefix=/usr/lib/postgresql/8.4 --with-pgport=5840 --with-python')
run('make')
env.safe_sudo('make install')
