def install_protvis(env):
""" Installs Andrew Brock's proteomics visualize tool.
https://bitbucket.org/Andrew_Brock/proteomics-visualise/
"""
_setup_protvis_env(env)
protvis_home = env["protvis_home"]
env.safe_sudo("sudo apt-get -y --force-yes install libxml2-dev libxslt-dev")
run("rm -rf protvis")
run("git clone -b lorikeet https://github.com/jmchilton/protvis.git")
with cd("protvis"):
run("git submodule init")
run("git submodule update")
env.safe_sudo("rsync -avur --delete-after . %s" % (protvis_home))
_chown_galaxy(env, protvis_home)
with cd(protvis_home):
env.safe_sudo("./setup.sh", user=env.get("galaxy_user", "galaxy"))
#default_revision = "8cc6af1c492c"
#
#revision = env.get("protvis_revision", default_revision)
#url = _get_bitbucket_download_url(revision, "https://bitbucket.org/Andrew_Brock/proteomics-visualise")
#def _make(env):
#_get_install(url, env, _make)
galaxy_data_dir = env.get('galaxy_data_dir', "/mnt/galaxyData/")
protvis_converted_files_dir = env.get('protvis_converted_files_dir')
_write_to_file('''GALAXY_ROOT = "%s"
PATH_WHITELIST = ["%s/files/", "%s"]
CONVERTED_FILES = "%s"
''' % (env.galaxy_home, galaxy_data_dir, protvis_converted_files_dir, protvis_converted_files_dir), "%s/conf.py" % protvis_home, 0755)
_setup_protvis_service(env)
def _clone_galaxy_repo(env):
"""
Clone Galaxy source code repository from ``env.galaxy_repository`` to
``env.galaxy_home``, setting the directory ownership to ``env.galaxy_user``
This method cannot be used to update an existing Galaxy installation.
"""
# Make sure ``env.galaxy_home`` dir exists but without Galaxy in it
galaxy_exists = False
if exists(env.galaxy_home):
if exists(os.path.join(env.galaxy_home, '.hg')):
env.logger.warning("Galaxy install dir {0} exists and seems to have "
"a Mercurial repository already there. Galaxy already installed?"
.format(env.galaxy_home))
galaxy_exists = True
else:
sudo("mkdir -p '%s'" % env.galaxy_home)
if not galaxy_exists:
with cd(env.galaxy_home):
# Needs to be done as non galaxy user, otherwise we have a
# permissions problem.
galaxy_repository = env.get("galaxy_repository", 'https://bitbucket.org/galaxy/galaxy-central/')
env.safe_sudo('hg clone %s .' % galaxy_repository)
# Make sure ``env.galaxy_home`` is owned by ``env.galaxy_user``
_chown_galaxy(env, env.galaxy_home)
# Make sure env.galaxy_home root dir is also owned by env.galaxy_user so Galaxy
# process can create necessary dirs (e.g., shed_tools, tmp)
sudo("chown {0}:{0} {1}".format(env.galaxy_user, os.path.split(env.galaxy_home)[0]))
# If needed, custom-configure this freshly cloned Galaxy
preconfigured = _read_boolean(env, "galaxy_preconfigured_repository", False)
if not preconfigured:
_configure_galaxy_repository(env)
def install_protvis(env):
""" Installs Andrew Brock's proteomics visualize tool.
https://bitbucket.org/Andrew_Brock/proteomics-visualise/
"""
_setup_protvis_env(env)
protvis_home = env["protvis_home"]
env.safe_sudo(
"sudo apt-get -y --force-yes install libxml2-dev libxslt-dev")
run("rm -rf protvis")
run("git clone -b lorikeet git://github.com/jmchilton/protvis.git")
with cd("protvis"):
run("git submodule init")
run("git submodule update")
env.safe_sudo("rsync -avur --delete-after . %s" % (protvis_home))
_chown_galaxy(env, protvis_home)
with cd(protvis_home):
env.safe_sudo("./setup.sh", user=env.get("galaxy_user", "galaxy"))
#default_revision = "8cc6af1c492c"
#
#revision = env.get("protvis_revision", default_revision)
#url = _get_bitbucket_download_url(revision, "https://bitbucket.org/Andrew_Brock/proteomics-visualise")
#def _make(env):
#_get_install(url, env, _make)
galaxy_data_dir = env.get('galaxy_data_dir', "/mnt/galaxyData/")
protvis_converted_files_dir = env.get('protvis_converted_files_dir')
_write_to_file(
'''GALAXY_ROOT = "%s"
PATH_WHITELIST = ["%s/files/", "%s"]
CONVERTED_FILES = "%s"
''' % (env.galaxy_home, galaxy_data_dir, protvis_converted_files_dir,
protvis_converted_files_dir), "%s/conf.py" % protvis_home, 0755)
_setup_protvis_service(env)
def _clone_galaxy_repo(env):
"""
Clone Galaxy source code repository from ``env.galaxy_repository`` to
``env.galaxy_home``, setting the directory ownership to ``env.galaxy_user``
This method cannot be used to update an existing Galaxy installation.
"""
# Make sure ``env.galaxy_home`` dir exists but without Galaxy in it
galaxy_exists = False
if exists(env.galaxy_home):
if exists(os.path.join(env.galaxy_home, '.hg')):
env.logger.warning("Galaxy install dir {0} exists and seems to have " \
"a Mercurial repository already there. Galaxy already installed?"\
.format(env.galaxy_home))
galaxy_exists = True
else:
sudo("mkdir -p '%s'" % env.galaxy_home)
if not galaxy_exists:
with cd(env.galaxy_home):
# Needs to be done as non galaxy user, otherwise we have a
# permissions problem.
galaxy_repository = env.get("galaxy_repository", 'https://bitbucket.org/galaxy/galaxy-central/')
env.safe_sudo('hg clone %s .' % galaxy_repository)
# Make sure ``env.galaxy_home`` is owned by ``env.galaxy_user``
_chown_galaxy(env, env.galaxy_home)
# Make sure env.galaxy_home root dir is also owned by env.galaxy_user so Galaxy
# process can create necessary dirs (e.g., shed_tools, tmp)
sudo("chown {0}:{0} {1}".format(env.galaxy_user, os.path.split(env.galaxy_home)[0]))
# If needed, custom-configure this freshly cloned Galaxy
preconfigured = _read_boolean(env, "galaxy_preconfigured_repository", False)
if not preconfigured:
_configure_galaxy_repository(env)
def _configure_galaxy_options(env, option_dict=None, prefix="galaxy_universe_"):
"""
Read through fabric's environment and make sure any property of
the form galaxy_universe_XXX=YYY, lands up in Galaxy's universe_wsgi.ini
options as XXX=YYY using John Chilton's configuration directory work:
https://bitbucket.org/galaxy/galaxy-central/pull-request/44/allow-usage-of-directory-of-configuration
Until, the above pull request is accepted, its changeset should be pulled
into the configured Galaxy repository.
"""
galaxy_conf_directory = env.get("galaxy_conf_directory", False)
if not galaxy_conf_directory:
return False
# By default just read the options from env (i.e. from fabricrc), but
# allow override so the options can come from a YAML file (such as done
# with galaxy-vm-launcher.)
if option_dict is None:
option_dict = env
option_priority = env.get("galaxy_conf_priority", "200")
for key, value in option_dict.iteritems():
if key.startswith(prefix):
key = key[len(prefix):]
conf_file_name = "%s_override_%s.ini" % (option_priority, key)
conf_file = os.path.join(galaxy_conf_directory, conf_file_name)
contents = "[app:main]\n%s=%s" % (key, value)
_write_to_file(contents, conf_file, 0700)
_chown_galaxy(env, conf_file)
def _configure_galaxy_options(env, option_dict=None, prefix="galaxy_universe_"):
"""
Read through fabric's environment and make sure any property of
the form galaxy_universe_XXX=YYY, lands up in Galaxy's universe_wsgi.ini
options as XXX=YYY using John Chilton's configuration directory work:
https://bitbucket.org/galaxy/galaxy-central/pull-request/44/allow-usage-of-directory-of-configuration
Until, the above pull request is accepted, its changeset should be pulled
into the configured Galaxy repository.
"""
galaxy_conf_directory = env.get("galaxy_conf_directory", False)
if not galaxy_conf_directory:
return False
# By default just read the options from env (i.e. from fabricrc), but
# allow override so the options can come from a YAML file (such as done
# with galaxy-vm-launcher.)
if option_dict is None:
option_dict = env
option_priority = env.get("galaxy_conf_priority", "200")
for key, value in option_dict.iteritems():
if key.startswith(prefix):
key = key[len(prefix):]
conf_file_name = "%s_override_%s.ini" % (option_priority, key)
conf_file = os.path.join(galaxy_conf_directory, conf_file_name)
contents = "[app:main]\n%s=%s" % (key, value)
_write_to_file(contents, conf_file, 0700)
_chown_galaxy(env, conf_file)
def _configure_galaxy_options(env,
option_dict=None,
prefix="galaxy_universe_"):
"""
Read through fabric's environment and make sure any property of
the form galaxy_universe_XXX=YYY lands up in Galaxy's universe_wsgi.ini
options as XXX=YYY using Galaxy configuration directory:
"""
galaxy_conf_directory = env.get("galaxy_conf_directory", None)
if not galaxy_conf_directory:
return False
# By default just read the options from env (i.e. from fabricrc), but
# allow override so the options can come from a YAML file (such as done
# with galaxy-vm-launcher.)
if option_dict is None:
option_dict = env
option_priority = env.get("galaxy_conf_priority", "200")
for key, value in option_dict.iteritems():
if key.startswith(prefix):
key = key[len(prefix):]
conf_file_name = "%s_override_%s.ini" % (option_priority, key)
conf_file = os.path.join(galaxy_conf_directory, conf_file_name)
contents = "[app:main]\n%s=%s" % (key, value)
_write_to_file(contents, conf_file, "0700")
_chown_galaxy(env, conf_file)
def install_protkgem(env):
"""This method installs Ira Cooke's ProtK framework for the galaxy user.
By default this will install ProtK from rubygems server, but if
env.protk_version is set to <version>@<url> (e.g.
1.1.5@https://bitbucket.org/iracooke/protk-working) the
gem will be cloned with hg and installed from source.
"""
_prep_galaxy(env)
default_version = "1.2.0"
version_and_revision = env.get("protk_version", default_version)
install_from_source = version_and_revision.find("@") > 0
# e.g. protk_version = 1.1.5@https://bitbucket.org/iracooke/protk-working
if install_from_source:
(version, revision) = version_and_revision.split("@")
url = _get_bitbucket_download_url(revision, "https://bitbucket.org/iracooke/protk")
else:
version = version_and_revision
ruby_version = "1.9.3"
force_rvm_install = False
with prefix("HOME=~%s" % env.galaxy_user):
def rvm_exec(env, cmd="", rvm_cmd="use"):
prefix = ". $HOME/.rvm/scripts/rvm; rvm %s %s; " % (rvm_cmd, ruby_version)
env.safe_sudo("%s %s" % (prefix, cmd), user=env.galaxy_user)
if not exists("$HOME/.rvm") or force_rvm_install:
env.safe_sudo("curl -L get.rvm.io | bash -s stable; source ~%s/.rvm/scripts/rvm" % (env.galaxy_user), user=env.galaxy_user)
rvm_exec(env, rvm_cmd="install")
if not install_from_source:
# Typical rubygem install
rvm_exec(env, "gem install --no-ri --no-rdoc protk -v %s" % version)
else:
with cd("~%s" % env.galaxy_user):
env.safe_sudo("rm -rf protk_source; hg clone '%s' protk_source" % url, user=env.galaxy_user)
rvm_exec(env, "cd protk_source; gem build protk.gemspec; gem install protk")
protk_properties = {}
## ProtK can set these up itself, should make that an option.
protk_properties["tpp_root"] = os.path.join(env.galaxy_tools_dir, "transproteomic_pipeline", "default")
protk_properties['openms_root'] = "/usr" # os.path.join(env.galaxy_tools_dir, "openms", "default", "bin")
### Assumes omssa, blast, and transproteomic_pipeline CBL galaxy installs.
protk_properties['omssa_root'] = os.path.join(env.galaxy_tools_dir, "omssa", "default", "bin")
protk_properties['blast_root'] = os.path.join(env.galaxy_tools_dir, "blast+", "default")
protk_properties['pwiz_root'] = os.path.join(env.galaxy_tools_dir, "transproteomic_pipeline", "default", "bin")
# Other properties: log_file, blast_root
env.safe_sudo("mkdir -p \"$HOME/.protk\"", user=env.galaxy_user)
env.safe_sudo("mkdir -p \"$HOME/.protk/Databases\"", user=env.galaxy_user)
_write_to_file(yaml.dump(protk_properties), "/home/%s/.protk/config.yml" % env.galaxy_user, 0755)
rvm_exec(env, "protk_setup.rb galaxyenv")
install_dir = os.path.join(env.galaxy_tools_dir, "protkgem", version)
env.safe_sudo("mkdir -p '%s'" % install_dir)
_chown_galaxy(env, install_dir)
env.safe_sudo('ln -s -f "$HOME/.protk/galaxy/env.sh" "%s/env.sh"' % install_dir, user=env.galaxy_user)
with cd(install_dir):
with cd(".."):
env.safe_sudo("ln -s -f '%s' default" % version)
def _configure_galaxy_repository(env):
"""
Custom-configure Galaxy repository. This is primarily targeted at a cloud
deployment.
mi-deployment would always edit the repository in this way, but
galaxy-vm-launcher requires the configured Galaxy repository to pull
in the changesets from https://bitbucket.org/jmchilton/cloud-galaxy-dist
which prebakes these modifications in.
"""
_chown_galaxy(
env, env.galaxy_home) # Make sure env.galaxy_user owns env.galaxy_home
with cd(env.galaxy_home): # and settings(warn_only=True):
# Make sure Galaxy runs in a new shell and does not
# inherit the environment by adding the '-ES' flag
# to all invocations of python within run.sh
sudo("sed -i 's/python .\//python -ES .\//g' run.sh",
user=env.galaxy_user)
if _read_boolean(env, "galaxy_cloud", True):
# Append DRMAA_LIBRARY_PATH in run.sh as well (this file will exist
# once SGE is installed - which happens at instance contextualization)
sudo(
"grep -q 'export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' run.sh; if [ $? -eq 1 ]; then sed -i '2 a export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' run.sh; fi",
user=env.galaxy_user)
# Upload the custom cloud welcome screen files
if not exists("%s/static/images/cloud.gif" % env.galaxy_home):
sudo(
"wget --output-document=%s/static/images/cloud.gif %s/cloud.gif"
% (env.galaxy_home, CDN_ROOT_URL),
user=env.galaxy_user)
if not exists("%s/static/images/cloud_txt.png" % env.galaxy_home):
sudo(
"wget --output-document=%s/static/images/cloud_text.png %s/cloud_text.png"
% (env.galaxy_home, CDN_ROOT_URL),
user=env.galaxy_user)
sudo(
"wget --output-document=%s/static/welcome.html %s/welcome.html"
% (env.galaxy_home, CDN_ROOT_URL),
user=env.galaxy_user)
# Set up the symlink for SAMTOOLS (remove this code once SAMTOOLS is converted to data tables)
if exists("%s/tool-data/sam_fa_indices.loc" % env.galaxy_home):
sudo("rm %s/tool-data/sam_fa_indices.loc" % env.galaxy_home,
user=env.galaxy_user)
# TODO: Is this really necessary here? Shouldn't the tools do this themselves?
# set up the jars directory for Java tools
if not exists(env.galaxy_jars_dir):
sudo("mkdir -p %s" % env.galaxy_jars_dir, user=env.galaxy_user)
srma_dir = os.path.join(env.galaxy_tools_dir, 'srma', 'default')
haploview_dir = os.path.join(env.galaxy_tools_dir, 'haploview',
'default')
picard_dir = os.path.join(env.galaxy_tools_dir, 'picard', 'default')
sudo('ln -s -f %s/srma.jar %s' % (srma_dir, env.galaxy_jars_dir),
user=env.galaxy_user)
sudo('ln -s -f %s/haploview.jar %s' %
(haploview_dir, env.galaxy_jars_dir),
user=env.galaxy_user)
sudo('ln -s -f %s/*.jar %s' % (picard_dir, env.galaxy_jars_dir),
user=env.galaxy_user)
return True
def install_protkgem(env):
"""This method installs Ira Cooke's ProtK framework for the galaxy user.
By default this will install ProtK from rubygems server, but if
env.protk_version is set to <version>@<url> (e.g.
1.1.5@https://bitbucket.org/iracooke/protk-working) the
gem will be cloned with hg and installed from source.
"""
_prep_galaxy(env)
default_version = "1.2.0"
version_and_revision = env.get("protk_version", default_version)
install_from_source = version_and_revision.find("@") > 0
# e.g. protk_version = 1.1.5@https://bitbucket.org/iracooke/protk-working
if install_from_source:
(version, revision) = version_and_revision.split("@")
url = _get_bitbucket_download_url(revision, "https://bitbucket.org/iracooke/protk")
else:
version = version_and_revision
ruby_version = "1.9.3"
force_rvm_install = False
with prefix("HOME=~%s" % env.galaxy_user):
def rvm_exec(env, cmd="", rvm_cmd="use"):
prefix = ". $HOME/.rvm/scripts/rvm; rvm %s %s; " % (rvm_cmd, ruby_version)
env.safe_sudo("%s %s" % (prefix, cmd), user=env.galaxy_user)
if not exists("$HOME/.rvm") or force_rvm_install:
env.safe_sudo("curl -L get.rvm.io | bash -s stable; source ~%s/.rvm/scripts/rvm" % (env.galaxy_user), user=env.galaxy_user)
rvm_exec(env, rvm_cmd="install")
if not install_from_source:
# Typical rubygem install
rvm_exec(env, "gem install --no-ri --no-rdoc protk -v %s" % version)
else:
with cd("~%s" % env.galaxy_user):
env.safe_sudo("rm -rf protk_source; hg clone '%s' protk_source" % url, user=env.galaxy_user)
rvm_exec(env, "cd protk_source; gem build protk.gemspec; gem install protk")
protk_properties = {}
## ProtK can set these up itself, should make that an option.
protk_properties["tpp_root"] = os.path.join(env.galaxy_tools_dir, "transproteomic_pipeline", "default")
protk_properties['openms_root'] = "/usr" # os.path.join(env.galaxy_tools_dir, "openms", "default", "bin")
### Assumes omssa, blast, and transproteomic_pipeline CBL galaxy installs.
protk_properties['omssa_root'] = os.path.join(env.galaxy_tools_dir, "omssa", "default", "bin")
protk_properties['blast_root'] = os.path.join(env.galaxy_tools_dir, "blast", "default")
protk_properties['pwiz_root'] = os.path.join(env.galaxy_tools_dir, "transproteomic_pipeline", "default", "bin")
# Other properties: log_file, blast_root
env.safe_sudo("mkdir -p \"$HOME/.protk\"", user=env.galaxy_user)
env.safe_sudo("mkdir -p \"$HOME/.protk/Databases\"", user=env.galaxy_user)
_write_to_file(yaml.dump(protk_properties), "/home/%s/.protk/config.yml" % env.galaxy_user, 0755)
rvm_exec(env, "protk_setup.rb galaxyenv")
install_dir = os.path.join(env.galaxy_tools_dir, "protkgem", version)
env.safe_sudo("mkdir -p '%s'" % install_dir)
_chown_galaxy(env, install_dir)
env.safe_sudo('ln -s -f "$HOME/.protk/galaxy/env.sh" "%s/env.sh"' % install_dir, user=env.galaxy_user)
with cd(install_dir):
with cd(".."):
env.safe_sudo("ln -s -f '%s' default" % version)
def _setup_install_dir(env):
"""Sets up install dir and ensures its owned by Galaxy"""
if not exists(env.galaxy_tools_dir):
sudo("mkdir -p %s" % env.galaxy_tools_dir)
_chown_galaxy(env, env.galaxy_tools_dir)
if not exists(env.galaxy_jars_dir):
sudo("mkdir -p %s" % env.galaxy_jars_dir)
_chown_galaxy(env, env.galaxy_jars_dir)
def _setup_install_dir(env):
"""Sets up install dir and ensures its owned by Galaxy"""
if not exists(env.galaxy_tools_dir):
sudo("mkdir -p %s" % env.galaxy_tools_dir)
_chown_galaxy(env, env.galaxy_tools_dir)
if not exists(env.galaxy_jars_dir):
sudo("mkdir -p %s" % env.galaxy_jars_dir)
_chown_galaxy(env, env.galaxy_jars_dir)
def configure_ssh_key(options):
if "galaxy_ssh_key" in options:
key_file = options["galaxy_ssh_key"]
sudo("mkdir -p /home/%s/.ssh" % (env.galaxy_user))
sudo("chmod 700 /home/%s/.ssh" % (env.galaxy_user))
put(local_path=key_file,
remote_path="/home/%s/.ssh/%s" % (env.galaxy_user, os.path.basename(key_file)),
use_sudo=True,
mode=0600)
_chown_galaxy(env, "/home/%s/.ssh" % env.galaxy_user)
def _configure_galaxy_repository(env):
"""
Custom-configure Galaxy repository. This is primarily targeted at a cloud
deployment.
mi-deployment would always edit the repository in this way, but
galaxy-vm-launcher requires the configured Galaxy repository to pull
in the changesets from https://bitbucket.org/jmchilton/cloud-galaxy-dist
which prebakes these modifications in.
"""
_chown_galaxy(env, env.galaxy_home) # Make sure env.galaxy_user owns env.galaxy_home
with cd(env.galaxy_home): # and settings(warn_only=True):
# Make sure Galaxy runs in a new shell and does not
# inherit the environment by adding the '-ES' flag
# to all invocations of python within run.sh
sudo("sed -i 's/python .\//python -ES .\//g' run.sh", user=env.galaxy_user)
if _read_boolean(env, "galaxy_cloud", True):
# Append DRMAA_LIBRARY_PATH in run.sh as well (this file will exist
# once SGE is installed - which happens at instance contextualization)
sudo(
"grep -q 'export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' run.sh; if [ $? -eq 1 ]; then sed -i '2 a export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' run.sh; fi",
user=env.galaxy_user,
)
# Upload the custom cloud welcome screen files
if not exists("%s/static/images/cloud.gif" % env.galaxy_home):
sudo(
"wget --output-document=%s/static/images/cloud.gif %s/cloud.gif" % (env.galaxy_home, CDN_ROOT_URL),
user=env.galaxy_user,
)
if not exists("%s/static/images/cloud_txt.png" % env.galaxy_home):
sudo(
"wget --output-document=%s/static/images/cloud_text.png %s/cloud_text.png"
% (env.galaxy_home, CDN_ROOT_URL),
user=env.galaxy_user,
)
sudo(
"wget --output-document=%s/static/welcome.html %s/welcome.html" % (env.galaxy_home, CDN_ROOT_URL),
user=env.galaxy_user,
)
# Set up the symlink for SAMTOOLS (remove this code once SAMTOOLS is converted to data tables)
if exists("%s/tool-data/sam_fa_indices.loc" % env.galaxy_home):
sudo("rm %s/tool-data/sam_fa_indices.loc" % env.galaxy_home, user=env.galaxy_user)
# TODO: Is this really necessary here? Shouldn't the tools do this themselves?
# set up the jars directory for Java tools
if not exists(env.galaxy_jars_dir):
sudo("mkdir -p %s" % env.galaxy_jars_dir, user=env.galaxy_user)
srma_dir = os.path.join(env.galaxy_tools_dir, "srma", "default")
haploview_dir = os.path.join(env.galaxy_tools_dir, "haploview", "default")
picard_dir = os.path.join(env.galaxy_tools_dir, "picard", "default")
sudo("ln -s %s/srma.jar %s" % (srma_dir, env.galaxy_jars_dir), user=env.galaxy_user)
sudo("ln -s %s/haploview.jar %s" % (haploview_dir, env.galaxy_jars_dir), user=env.galaxy_user)
sudo("ln -s %s/*.jar %s" % (picard_dir, env.galaxy_jars_dir), user=env.galaxy_user)
return True
def copy_runtime_properties(fqdn, options):
runtime_properties_raw = options.get("runtime_properties", {})
runtime_properties = {"FQDN": fqdn}
for runtime_property_raw in runtime_properties_raw:
(name, value) = runtime_property_raw.split(":")
runtime_properties[name] = value
export_file = ""
for (name, value) in runtime_properties.iteritems():
export_file = "export %s=%s\n%s" % (name, value, export_file)
sudo('mkdir -p %s' % env.galaxy_home)
_chown_galaxy(env, env.galaxy_home)
sudo("echo '%s' > %s/runtime_properties" % (export_file, env.galaxy_home), user=env.galaxy_user)
def copy_runtime_properties(fqdn, options):
runtime_properties_raw = options.get("runtime_properties", {})
runtime_properties = {"FQDN": fqdn}
for runtime_property_raw in runtime_properties_raw:
(name, value) = runtime_property_raw.split(":")
runtime_properties[name] = value
export_file = ""
for (name, value) in runtime_properties.iteritems():
export_file = "export %s=%s\n%s" % (name, value, export_file)
sudo('mkdir -p %s' % env.galaxy_home)
_chown_galaxy(env, env.galaxy_home)
sudo("echo '%s' > %s/runtime_properties" % (export_file, env.galaxy_home),
user=env.galaxy_user)
def _setup_trackster(env):
"""
Download .len files required by Trackster:
http://wiki.galaxyproject.org/Learn/Visualization#Setup_for_Local_Instances
"""
if not exists(env.galaxy_len_files):
sudo("mkdir -p {0}".format(env.galaxy_len_files))
with cd(env.galaxy_len_files):
local_fn = "len-files.tar.gz"
sudo('wget --output-document={0} '
'https://s3.amazonaws.com/usegalaxy/len-files.tar.gz'.format(local_fn))
sudo("tar xzf {0}".format(local_fn))
sudo("rm {0}".format(local_fn))
_chown_galaxy(env, env.galaxy_len_files)
def _install_galaxy(env):
""" Used to install Galaxy and setup its environment.
This method cannot be used to update an existing instance of Galaxy code;
see volume_manipulations_fab.py script for that functionality.
Also, this method is somewhat targeted for the EC2 deployment so some
tweaking of the code may be desirable."""
# MP: we need to have a tmp directory available if files already exist
# in the galaxy install directory
install_cmd = sudo if env.get("use_sudo", True) else run
base_tmp_dir = env.get("galaxy_tmp_dir", "/mnt")
tmp_dir = os.path.join(base_tmp_dir, "fab_tmp")
if exists(tmp_dir):
install_cmd("rm -rf %s" % tmp_dir)
if exists(env.galaxy_home):
if exists(os.path.join(env.galaxy_home, '.hg')):
env.logger.info("Galaxy install dir '%s' exists and seems to have a Mercurial repository already there. Galaxy already installed.")
return False
else:
# MP: need to move any files already in galaxy home so that hg
# can checkout files.
if not exists(tmp_dir):
install_cmd("mkdir %s" % tmp_dir)
install_cmd("chown %s %s" % (env.user, tmp_dir))
install_cmd("mv %s/* %s" % (env.galaxy_home, tmp_dir))
## This is slightly different than mi-deployment to handle the
## case when the bucket url doesn't match the desired path.
if not exists(env.galaxy_home):
sudo("mkdir -p '%s'" % env.galaxy_home)
_chown_galaxy(env, env.galaxy_home)
with cd(env.galaxy_home):
# MP needs to be done as non galaxy user, otherwise we have a
# permissions problem.
galaxy_repository = env.get("galaxy_repository", 'https://bitbucket.org/galaxy/galaxy-central/')
sudo('hg clone %s .' % galaxy_repository)
# MP: now we need to move the files back into the galaxy directory.
if exists(tmp_dir):
install_cmd("cp -R %s/* %s" % (tmp_dir, env.galaxy_home))
install_cmd("rm -rf %s" % tmp_dir)
# MP: Ensure that everything under install dir is owned by env.galaxy_user
sudo("chown --recursive %s:%s %s"
% (env.galaxy_user, env.galaxy_user, os.path.split(env.galaxy_tools_dir)[0]))
sudo("chmod 755 %s" % os.path.split(env.galaxy_tools_dir)[0])
env.get("galaxy_preconfigured_repository", False)
preconfigured = _read_boolean(env, "galaxy_preconfigured_repository", False)
if not preconfigured:
_configure_galaxy_repository(env)
_install_tools(env)
return True
def _configure_galaxy_repository(env):
""" mi-deployment would always edit the repository in this way, but
galaxy-vm-launcher requires the configured Galaxy repository to pull
in the changesets from https://bitbucket.org/jmchilton/cloud-galaxy-dist
which prebakes these modifications in.
"""
sudo("chown -R %s %s" % (env.galaxy_user, env.galaxy_home))
with cd(env.galaxy_home): # and settings(warn_only=True):
# Make sure Galaxy runs in a new shell and does not
# inherit the environment by adding the '-ES' flag
# to all invocations of python within run.sh
sudo("sed -i 's/python .\//python -ES .\//g' run.sh", user=env.galaxy_user)
if _read_boolean(env, "galaxy_cloud", True):
# Append DRMAA_LIBRARY_PATH in run.sh as well (this file will exist
# once SGE is installed - which happens at instance contextualization)
sudo("grep -q 'export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' run.sh; if [ $? -eq 1 ]; then sed -i '2 a export DRMAA_LIBRARY_PATH=/opt/sge/lib/lx24-amd64/libdrmaa.so.1.0' run.sh; fi", user=env.galaxy_user)
# Upload the custom cloud welcome screen files
if not exists("%s/static/images/cloud.gif" % env.galaxy_home):
sudo("wget --output-document=%s/static/images/cloud.gif %s/cloud.gif" % (env.galaxy_home, CDN_ROOT_URL), user=env.galaxy_user)
if not exists("%s/static/images/cloud_txt.png" % env.galaxy_home):
sudo("wget --output-document=%s/static/images/cloud_text.png %s/cloud_text.png" % (env.galaxy_home, CDN_ROOT_URL), user=env.galaxy_user)
sudo("wget --output-document=%s/static/welcome.html %s/welcome.html" % (env.galaxy_home, CDN_ROOT_URL), user=env.galaxy_user)
# Set up the symlink for SAMTOOLS (remove this code once SAMTOOLS is converted to data tables)
if exists("%s/tool-data/sam_fa_indices.loc" % env.galaxy_home):
sudo("rm %s/tool-data/sam_fa_indices.loc" % env.galaxy_home, user=env.galaxy_user)
tmp_loc = False
if not exists(env.galaxy_loc_files):
sudo("mkdir -p '%s'" % env.galaxy_loc_files)
_chown_galaxy(env, env.galaxy_loc_files)
if not exists("%s/sam_fa_indices.loc" % env.galaxy_loc_files):
sudo("touch %s/sam_fa_indices.loc" % env.galaxy_loc_files, user=env.galaxy_user)
tmp_loc = True
sudo("ln -s %s/sam_fa_indices.loc %s/tool-data/sam_fa_indices.loc" % (env.galaxy_loc_files, env.galaxy_home), user=env.galaxy_user)
if tmp_loc:
sudo("rm %s/sam_fa_indices.loc" % env.galaxy_loc_files, user=env.galaxy_user)
# set up the special HYPHY link in tool-data/
hyphy_dir = os.path.join(env.galaxy_tools_dir, 'hyphy', 'default')
sudo('ln -s %s tool-data/HYPHY' % hyphy_dir, user=env.galaxy_user)
# set up the jars directory for Java tools
if not exists('tool-data/shared/jars'):
sudo("mkdir -p tool-data/shared/jars", user=env.galaxy_user)
srma_dir = os.path.join(env.galaxy_tools_dir, 'srma', 'default')
haploview_dir = os.path.join(env.galaxy_tools_dir, 'haploview', 'default')
picard_dir = os.path.join(env.galaxy_tools_dir, 'picard', 'default')
sudo('ln -s %s/srma.jar tool-data/shared/jars/.' % srma_dir, user=env.galaxy_user)
sudo('ln -s %s/haploview.jar tool-data/shared/jars/.' % haploview_dir, user=env.galaxy_user)
sudo('ln -s %s/*.jar tool-data/shared/jars/.' % picard_dir, user=env.galaxy_user)
return True
def _install_tools(env, tools_conf=None):
"""
Install tools needed for Galaxy along with tool configuration
directories needed by Galaxy.
"""
if not tools_conf:
tools_conf = _load_tools_conf(env)
if _read_boolean(env, "galaxy_install_dependencies", False):
# Need to ensure the install dir exists and is owned by env.galaxy_user
_setup_install_dir(env)
_install_applications(env, tools_conf)
_chown_galaxy(env, env.galaxy_tools_dir)
_chown_galaxy(env, env.galaxy_jars_dir)
if _read_boolean(env, "galaxy_install_r_packages", False):
_install_r_packages(tools_conf)
def _install_tools(env, tools_conf=None):
"""
Install tools needed for Galaxy along with tool configuration
directories needed by Galaxy.
"""
if not tools_conf:
tools_conf = _load_tools_conf(env)
if _read_boolean(env, "galaxy_install_dependencies", False):
# Need to ensure the install dir exists and is owned by env.galaxy_user
_setup_install_dir(env)
_install_applications(env, tools_conf)
_chown_galaxy(env, env.galaxy_tools_dir)
_chown_galaxy(env, env.galaxy_jars_dir)
if _read_boolean(env, "galaxy_install_r_packages", False):
_install_r_packages(tools_conf)
def _install_galaxy(env):
"""
Used to install Galaxy and setup its environment, including tools.
This method is somewhat targeted for the cloud deployment so some
tweaking of the code may be desirable. This method cannot be used
to update an existing Galaxy.
"""
_clone_galaxy_repo(env)
_chown_galaxy(env, env.galaxy_home) # Make sure env.galaxy_user owns env.galaxy_home
sudo("chmod 755 %s" % os.path.split(env.galaxy_home)[0])
setup_db = _read_boolean(env, "galaxy_setup_database", False)
if setup_db:
_setup_galaxy_db(env)
setup_service = _read_boolean(env, "galaxy_setup_service", False)
if setup_service:
_setup_service(env)
_setup_trackster(env)
_setup_shed_tools_dir(env)
_install_tools(env)
setup_xvfb = _read_boolean(env, "galaxy_setup_xvfb", False)
if setup_xvfb:
_setup_xvfb(env)
_chown_galaxy(env, env.galaxy_home) # Make sure env.galaxy_user owns env.galaxy_home
_chown_galaxy(env, env.galaxy_loc_files) # Make sure env.galaxy_user owns env.galaxy_loc_files
return True
def install_protk(env):
"""Installs Ira Cooke's ProtK framework for the
galaxy user"""
_prep_galaxy(env)
default_version = "0.95@e81050c1c658"
version_and_revision = env.get("protk_version", default_version)
(version, revision) = version_and_revision.split("@")
url = _get_bitbucket_download_url(revision, "https://bitbucket.org/iracooke/protk")
with prefix("HOME=~%s" % env.galaxy_user):
if not exists("$HOME/.rvm"):
env.safe_sudo("curl -L get.rvm.io | bash -s stable; source ~%s/.rvm/scripts/rvm" % (env.galaxy_user), user=env.galaxy_user)
env.safe_sudo(". $HOME/.rvm/scripts/rvm; rvm install 1.8.7 | cat", user=env.galaxy_user)
env.safe_sudo(". $HOME/.rvm/scripts/rvm; rvm 1.8.7 do gem install rake --no-rdoc --no-ri", user=env.galaxy_user)
install_dir = os.path.join(env.galaxy_tools_dir, "protk", version)
def _make(env):
env.safe_sudo("PROTK_DIR=%s; rm -rf $PROTK_DIR; mkdir -p $PROTK_DIR; mv * $PROTK_DIR" % (install_dir))
_get_install(url, env, _make)
_chown_galaxy(env, install_dir)
with prefix("HOME=~%s" % env.galaxy_user):
import StringIO
output = StringIO.StringIO()
get("%s/config.yml.sample" % install_dir, output)
sample_config = yaml.load(output.getvalue())
sample_config['test']['tpp_bin'] = os.path.join(env.galaxy_tools_dir, "transproteomic_pipeline", "default", "bin")
sample_config['test']['omssa_bin'] = os.path.join(env.galaxy_tools_dir, "omssa", "default", "bin")
sample_config['test']['ncbi_tools_bin'] = os.path.join(env.galaxy_tools_dir, "blast", "default", "bin")
sample_config['test']['openms_bin'] = os.path.join(env.galaxy_tools_dir, "openms", "default", "bin")
#sample_config['test']['galaxy_root']=env.galaxy_home
_write_to_file(yaml.dump(sample_config), "%s/config.yml" % install_dir, 0755)
with cd(install_dir):
#env.safe_sudo("mv /tmp/config.yaml .")
env.safe_sudo(". $HOME/.rvm/scripts/rvm; echo No | ./setup.sh", user=env.galaxy_user)
env.safe_sudo("gcc -o make_random make_random.c -lm; ln -s ../make_random bin", user=env.galaxy_user)
with cd("%s/.." % install_dir):
env.safe_sudo("ln -s -f %s default" % version)
def _clone_galaxy_repo(env):
# MP: we need to have a tmp directory available if files already exist
# in the galaxy install directory
install_cmd = sudo if env.get("use_sudo", True) else run
base_tmp_dir = env.get("galaxy_tmp_dir", "/mnt")
tmp_dir = os.path.join(base_tmp_dir, "fab_tmp")
if exists(tmp_dir):
install_cmd("rm -rf %s" % tmp_dir)
if exists(env.galaxy_home):
if exists(os.path.join(env.galaxy_home, '.hg')):
env.logger.info("Galaxy install dir '%s' exists and seems to have a Mercurial repository already there. Galaxy already installed.")
return False
else:
# MP: need to move any files already in galaxy home so that hg
# can checkout files.
if not exists(tmp_dir):
install_cmd("mkdir %s" % tmp_dir)
install_cmd("chown %s %s" % (env.user, tmp_dir))
install_cmd("mv %s/* %s" % (env.galaxy_home, tmp_dir))
## This is slightly different than mi-deployment to handle the
## case when the bucket url doesn't match the desired path.
if not exists(env.galaxy_home):
sudo("mkdir -p '%s'" % env.galaxy_home)
_chown_galaxy(env, env.galaxy_home)
with cd(env.galaxy_home):
# MP needs to be done as non galaxy user, otherwise we have a
# permissions problem.
galaxy_repository = env.get("galaxy_repository", 'https://bitbucket.org/galaxy/galaxy-central/')
sudo('hg clone %s .' % galaxy_repository)
# MP: now we need to move the files back into the galaxy directory.
if exists(tmp_dir):
install_cmd("cp -R %s/* %s" % (tmp_dir, env.galaxy_home))
install_cmd("rm -rf %s" % tmp_dir)
preconfigured = _read_boolean(env, "galaxy_preconfigured_repository", False)
if not preconfigured:
_configure_galaxy_repository(env)
def _clone_galaxy_repo(env):
# MP: we need to have a tmp directory available if files already exist
# in the galaxy install directory
install_cmd = sudo if env.get("use_sudo", True) else run
base_tmp_dir = env.get("galaxy_tmp_dir", "/mnt")
tmp_dir = os.path.join(base_tmp_dir, "fab_tmp")
if exists(tmp_dir):
install_cmd("rm -rf %s" % tmp_dir)
if exists(env.galaxy_home):
if exists(os.path.join(env.galaxy_home, '.hg')):
env.logger.info("Galaxy install dir '%s' exists and seems to have a Mercurial repository already there. Galaxy already installed.")
return False
else:
# MP: need to move any files already in galaxy home so that hg
# can checkout files.
if not exists(tmp_dir):
install_cmd("mkdir %s" % tmp_dir)
install_cmd("chown %s %s" % (env.user, tmp_dir))
install_cmd("mv %s/* %s" % (env.galaxy_home, tmp_dir))
## This is slightly different than mi-deployment to handle the
## case when the bucket url doesn't match the desired path.
if not exists(env.galaxy_home):
sudo("mkdir -p '%s'" % env.galaxy_home)
_chown_galaxy(env, env.galaxy_home)
with cd(env.galaxy_home):
# MP needs to be done as non galaxy user, otherwise we have a
# permissions problem.
galaxy_repository = env.get("galaxy_repository", 'https://bitbucket.org/galaxy/galaxy-central/')
sudo('hg clone %s .' % galaxy_repository)
# MP: now we need to move the files back into the galaxy directory.
if exists(tmp_dir):
install_cmd("cp -R %s/* %s" % (tmp_dir, env.galaxy_home))
install_cmd("rm -rf %s" % tmp_dir)
preconfigured = _read_boolean(env, "galaxy_preconfigured_repository", False)
if not preconfigured:
_configure_galaxy_repository(env)
def _configure_galaxy_options(env, option_dict=None, prefix="galaxy_universe_"):
"""
Read through fabric's environment and make sure any property of
the form galaxy_universe_XXX=YYY lands up in Galaxy's universe_wsgi.ini
options as XXX=YYY using Galaxy configuration directory:
"""
galaxy_conf_directory = env.get("galaxy_conf_directory", None)
if not galaxy_conf_directory:
return False
# By default just read the options from env (i.e. from fabricrc), but
# allow override so the options can come from a YAML file (such as done
# with galaxy-vm-launcher.)
if option_dict is None:
option_dict = env
option_priority = env.get("galaxy_conf_priority", "200")
for key, value in option_dict.iteritems():
if key.startswith(prefix):
key = key[len(prefix):]
conf_file_name = "%s_override_%s.ini" % (option_priority, key)
conf_file = os.path.join(galaxy_conf_directory, conf_file_name)
contents = "[app:main]\n%s=%s" % (key, value)
_write_to_file(contents, conf_file, 0700)
_chown_galaxy(env, conf_file)
def _setup_install_dir(env):
"""Sets up install dir and ensures its owned by Galaxy"""
if not exists(env.galaxy_tools_dir):
sudo("mkdir -p %s" % env.galaxy_tools_dir)
_chown_galaxy(env, env.galaxy_tools_dir)
# Create a general-purpose ``bin`` directory under the galaxy_tools_dir
# and put it on the PATH so users can more easily add custom tools
bin_dir = os.path.join(env.galaxy_tools_dir, 'bin')
if not exists(bin_dir):
sudo("mkdir -p %s" % bin_dir)
_chown_galaxy(env, bin_dir)
line = "export PATH={0}:$PATH".format(bin_dir)
_add_to_profiles(line)
if not exists(env.galaxy_jars_dir):
sudo("mkdir -p %s" % env.galaxy_jars_dir)
_chown_galaxy(env, env.galaxy_jars_dir)
def _setup_install_dir(env):
"""Sets up install dir and ensures its owned by Galaxy"""
if not exists(env.galaxy_tools_dir):
sudo("mkdir -p %s" % env.galaxy_tools_dir)
_chown_galaxy(env, env.galaxy_tools_dir)
# Create a general-purpose ``bin`` directory under the galaxy_tools_dir
# and put it on the PATH so users can more easily add custom tools
bin_dir = os.path.join(env.galaxy_tools_dir, 'bin')
if not exists(bin_dir):
sudo("mkdir -p %s" % bin_dir)
_chown_galaxy(env, bin_dir)
line = "export PATH={0}:$PATH".format(bin_dir)
_add_to_profiles(line)
if not exists(env.galaxy_jars_dir):
sudo("mkdir -p %s" % env.galaxy_jars_dir)
_chown_galaxy(env, env.galaxy_jars_dir)
def _make(env):
env.safe_sudo("rsync -avur --delete-after . %s" % (protvis_home))
_chown_galaxy(env, protvis_home)
with cd(protvis_home):
env.safe_sudo("./setup.sh", user=env.get("galaxy_user", "galaxy"))
def _start_nginx(env):
galaxy_data = env.galaxy_data_mount
env.safe_sudo("mkdir -p '%s'" % env.galaxy_data)
_chown_galaxy(env, galaxy_data)
start_service("nginx")
def _make_dir_for_galaxy(env, path):
sudo("mkdir -p '%s'" % path)
_chown_galaxy(env, path)
def _start_nginx(env):
galaxy_data = env.galaxy_data_mount
env.safe_sudo("mkdir -p '%s'" % env.galaxy_data)
_chown_galaxy(env, galaxy_data)
start_service("nginx")