def _setup_xvfb(env):
_setup_conf_file(env, "/etc/init.d/xvfb", "xvfb_init", default_source="xvfb_init")
_setup_conf_file(env, "/etc/default/xvfb", "xvfb_default", default_source="xvfb_default")
_setup_simple_service("xvfb")
env.safe_sudo("mkdir /var/lib/xvfb; chown root:root /var/lib/xvfb; chmod 0755 /var/lib/xvfb")
display_export = "-v DIPSLAY=:42"
_add_to_profiles(display_export, profiles=["/home/%s/.sge_request" % env.get("galaxy_user", "galaxy")])
def _setup_env(env):
"""
Setup the system environment required to run CloudMan. This means
installing required system-level packages (as defined in CBL's
``packages.yaml``, or a flavor thereof) and Python dependencies
(i.e., libraries) as defined in CloudMan's ``requirements.txt`` file.
"""
# Get and install required system packages
if env.distribution in ["debian", "ubuntu"]:
config_file = get_config_file(env, "packages.yaml")
(packages, _) = _yaml_to_packages(config_file.base, 'cloudman')
# Allow flavors to modify the package list
packages = env.flavor.rewrite_config_items("packages", packages)
_setup_apt_automation()
_apt_packages(pkg_list=packages)
elif env.distribution in ["centos", "scientificlinux"]:
env.logger.warn("No CloudMan system package dependencies for CentOS")
pass
# Get and install required Python libraries
with _make_tmp_dir() as work_dir:
with cd(work_dir):
url = os.path.join(CM_REPO_ROOT_URL, 'requirements.txt')
_create_python_virtualenv(env, 'CM', reqs_url=url)
# Add a custom vimrc
vimrc_url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', 'vimrc')
remote_file = '/etc/vim/vimrc'
if env.safe_exists("/etc/vim"):
env.safe_sudo("wget --output-document=%s %s" % (remote_file, vimrc_url))
env.logger.debug("Added a custom vimrc to {0}".format(remote_file))
# Setup profile
aliases = ['alias lt="ls -ltr"', 'alias ll="ls -l"']
for alias in aliases:
_add_to_profiles(alias, ['/etc/bash.bashrc'])
env.logger.info("Done setting up CloudMan's environment")
def _setup_xvfb(env):
_setup_conf_file(env, "/etc/init.d/xvfb", "xvfb_init", default_source="xvfb_init")
_setup_conf_file(env, "/etc/default/xvfb", "xvfb_default", default_source="xvfb_default")
_setup_simple_service("xvfb")
env.safe_sudo("mkdir /var/lib/xvfb; chown root:root /var/lib/xvfb; chmod 0755 /var/lib/xvfb")
display_export = "-v DIPSLAY=:42"
_add_to_profiles(display_export, profiles=["/home/%s/.sge_request" % env.get("galaxy_user", "galaxy")])
def _install_galaxy_config(tool_env, bin_dirs, env_vars):
"""
Setup galaxy tool config files (env.sh-es) and default version
symbolic links.
"""
install_dir = tool_env["system_install"]
env_path = os.path.join(install_dir, "env.sh")
bin_paths = [os.path.join(install_dir, bin_dir) for bin_dir in bin_dirs]
path_pieces = [bin_path for bin_path in bin_paths if exists(bin_path)]
if len(path_pieces) > 0 and not exists(env_path):
path_addtion = ":".join(path_pieces)
# Standard bin install, just add it to path
sudo("echo 'PATH=%s:$PATH' > %s" % (path_addtion, env_path))
venv_path = "%s/%s" % (install_dir, "venv")
if exists(venv_path):
# Have env.sh activate virtualdirectory
sudo("echo '. %s/bin/activate' >> %s" % (venv_path, env_path))
sudo("chmod +x %s" % env_path)
for env_var, env_var_value in env_vars.iteritems():
env_var_template = Template(env_var_value)
expanded_env_var_value = env_var_template.substitute(tool_env)
sudo("echo 'export %s=%s' >> %s" %
(env_var, expanded_env_var_value, env_path))
env.logger.debug("Added Galaxy env.sh file: %s" % env_path)
_set_default_config(tool_env, install_dir)
if _read_boolean(tool_env, "autoload_galaxy_tools",
False) and exists(env_path):
# In this case, the web user (e.g. ubuntu) should auto-load all of
# galaxy's default env.sh files so they are available for direct use
# as well.
_add_to_profiles(". %s" % env_path, profiles=["~/.bashrc"])
def _setup_env(env):
"""
Setup the system environment required to run CloudMan. This means
installing required system-level packages (as defined in CBL's
``packages.yaml``, or a flavor thereof) and Python dependencies
(i.e., libraries) as defined in CloudMan's ``requirements.txt`` file.
"""
# Get and install required system packages
if env.distribution in ["debian", "ubuntu"]:
config_file = get_config_file(env, "packages.yaml")
(packages, _) = _yaml_to_packages(config_file.base, 'cloudman')
# Allow editions and flavors to modify the package list
packages = env.edition.rewrite_config_items("packages", packages)
packages = env.flavor.rewrite_config_items("packages", packages)
_setup_apt_automation()
_apt_packages(pkg_list=packages)
elif env.distribution in ["centos", "scientificlinux"]:
env.logger.warn("No CloudMan system package dependencies for CentOS")
pass
# Get and install required Python libraries
with _make_tmp_dir() as work_dir:
with cd(work_dir):
url = os.path.join(CM_REPO_ROOT_URL, 'requirements.txt')
_create_python_virtualenv(env, 'CM', reqs_url=url)
# Add a custom vimrc
vimrc_url = os.path.join(MI_REPO_ROOT_URL, 'conf_files', 'vimrc')
remote_file = '/etc/vim/vimrc'
sudo("wget --output-document=%s %s" % (remote_file, vimrc_url))
env.logger.debug("Added a custom vimrc to {0}".format(remote_file))
# Setup profile
aliases = ['alias lt="ls -ltr"', 'alias ll="ls -l"']
for alias in aliases:
_add_to_profiles(alias, ['/etc/bash.bashrc'])
env.logger.info("Done setting up CloudMan's environment")
def _setup_env(self):
"""
Setup a custom user-level env
"""
# Add commond directories to PATH
path_additions = ("export PATH=/usr/lib/postgresql/9.1/bin:" +
"/usr/nginx/sbin:/mnt/galaxy/tools/bin:$PATH")
env.logger.debug("Amending the PATH with {0}".format(path_additions))
_add_to_profiles(path_additions, ['/etc/bash.bashrc'])
# Seed the history with frequently used commands
env.logger.debug("Setting bash history")
local = os.path.join(env.config_dir, os.pardir, "installed_files",
"bash_history")
remote = os.path.join('/home', 'ubuntu', '.bash_history')
put(local, remote, mode=0660, use_sudo=True)
# Install ipython profiles
users = ['ubuntu', 'galaxy']
for user in users:
env.logger.debug("Setting installing ipython profile for user {0}"
.format(user))
env.safe_sudo("su - {0} -c 'ipython profile create'".format(user))
local = os.path.join(env.config_dir, os.pardir, "installed_files",
"ipython_config.py")
remote = os.path.join('/home', user, '.ipython', 'profile_default',
"ipython_config.py")
put(local, remote, mode=0644, use_sudo=True)
env.safe_sudo("chown {0}:{0} {1}".format(user, remote))
def _install_galaxy_config(tool_env, bin_dirs, env_vars):
"""
Setup galaxy tool config files (env.sh-es) and default version
symbolic links.
"""
install_dir = tool_env["system_install"]
env_path = os.path.join(install_dir, "env.sh")
bin_paths = [os.path.join(install_dir, bin_dir) for bin_dir in bin_dirs]
path_pieces = [bin_path for bin_path in bin_paths if env.safe_exists(bin_path)]
if len(path_pieces) > 0 and not env.safe_exists(env_path):
path_addtion = ":".join(path_pieces)
# Standard bin install, just add it to path
env.safe_sudo("echo 'PATH=%s:$PATH' > %s" % (path_addtion, env_path))
venv_path = "%s/%s" % (install_dir, "venv")
if env.safe_exists(venv_path):
# Have env.sh activate virtualdirectory
env.safe_sudo("echo '. %s/bin/activate' >> %s" % (venv_path, env_path))
env.safe_sudo("chmod +x %s" % env_path)
for env_var, env_var_value in env_vars.iteritems():
env_var_template = Template(env_var_value)
expanded_env_var_value = env_var_template.substitute(tool_env)
env.safe_sudo("echo 'export %s=%s' >> %s" % (env_var, expanded_env_var_value, env_path))
env.logger.debug("Added Galaxy env.sh file: %s" % env_path)
# TODO: If a direct install (i.e. tool_install_dir specified instead of galaxy_tools_dir)
# default is still setup. This is not really desired.
_set_default_config(tool_env, install_dir)
if _read_boolean(tool_env, "autoload_galaxy_tools", False) and env.safe_exists(env_path):
# In this case, the web user (e.g. ubuntu) should auto-load all of
# galaxy's default env.sh files so they are available for direct use
# as well.
_add_to_profiles(". %s" % env_path, profiles=["~/.bashrc"])
def _setup_env(self):
"""
Setup a custom user-level env
"""
# Add commond directories to PATH
path_additions = ("export PATH=/usr/lib/postgresql/9.1/bin:" +
"/usr/nginx/sbin:/mnt/galaxy/tools/bin:$PATH")
env.logger.debug("Amending the PATH with {0}".format(path_additions))
_add_to_profiles(path_additions, ['/etc/bash.bashrc'])
# Seed the history with frequently used commands
env.logger.debug("Setting bash history")
local = os.path.join(env.config_dir, os.pardir, "installed_files",
"bash_history")
remote = os.path.join('/home', 'ubuntu', '.bash_history')
put(local, remote, mode=0660, use_sudo=True)
# Install ipython profiles
users = ['ubuntu', 'galaxy']
for user in users:
env.logger.debug(
"Setting installing ipython profile for user {0}".format(user))
env.safe_sudo("su - {0} -c 'ipython profile create'".format(user))
local = os.path.join(env.config_dir, os.pardir, "installed_files",
"ipython_config.py")
remote = os.path.join('/home', user, '.ipython', 'profile_default',
"ipython_config.py")
put(local, remote, mode=0644, use_sudo=True)
env.safe_sudo("chown {0}:{0} {1}".format(user, remote))
def _create_galaxy_db(env):
"""
Create a new PostgreSQL database for use by Galaxy
"""
c = _get_galaxy_db_configs(env)
if not env.safe_exists(c['psql_data_dir']):
env.safe_sudo("mkdir -p {0}".format(c['psql_data_dir']))
env.safe_sudo("chown --recursive {0}:{0} {1}".format(
c['psql_user'], c['psql_data_dir']))
# Initialize a new database for Galaxy in ``psql_data_dir``
if _dir_is_empty(c['psql_data_dir']):
env.safe_sudo("{0} -D {1}".format(
os.path.join(c['psql_bin_dir'], 'initdb'), c['psql_data_dir']),
user=c['psql_user'])
# Set port for the database server
env.safe_sed(c['psql_conf_file'],
'#port = 5432',
'port = {0}'.format(c['psql_port']),
use_sudo=True)
env.safe_sudo("chown {0}:{0} {1}".format(c['psql_user'],
c['psql_conf_file']))
# Start PostgreSQL server so a role for Galaxy user can be created
if not _postgres_running(env):
env.safe_sudo(c['pg_start_cmd'], user=c['psql_user'])
started = True
else:
# Restart is required so port setting takes effect
env.safe_sudo("{0} -D {1} -w -l {2} restart".format(c['pg_ctl_cmd']),
c['psql_data_dir'],
c['psql_log'],
user=c['psql_user'])
started = False
# Create a role for env.galaxy_user
env.safe_sudo('{0} -c"CREATE ROLE {1} LOGIN CREATEDB"'.format(
c['psql_cmd'], env.galaxy_user),
user=c['psql_user'])
# Create a Galaxy database
env.safe_sudo('{0} -p {1} {2}'.format(
os.path.join(c['psql_bin_dir'], 'createdb'), c['psql_port'],
c['galaxy_db_name']),
user=env.galaxy_user)
# Create a role for 'galaxyftp' user
env.safe_sudo(
'{0} -c"CREATE ROLE galaxyftp LOGIN PASSWORD \'{1}\'"'.format(
c['psql_cmd'], c['galaxy_ftp_user_pwd']),
user=c['psql_user'])
if started:
with settings(warn_only=True):
env.safe_sudo("{0}".format(c['pg_stop_cmd']), user=c['psql_user'])
exp = "export PATH={0}:$PATH".format(c['psql_bin_dir'])
_add_to_profiles(exp)
def _setup_install_dir(env):
"""Sets up install dir and ensures its owned by Galaxy"""
if not exists(env.galaxy_tools_dir):
sudo("mkdir -p %s" % env.galaxy_tools_dir)
_chown_galaxy(env, env.galaxy_tools_dir)
# Create a general-purpose ``bin`` directory under the galaxy_tools_dir
# and put it on the PATH so users can more easily add custom tools
bin_dir = os.path.join(env.galaxy_tools_dir, 'bin')
if not exists(bin_dir):
sudo("mkdir -p %s" % bin_dir)
_chown_galaxy(env, bin_dir)
line = "export PATH={0}:$PATH".format(bin_dir)
_add_to_profiles(line)
if not exists(env.galaxy_jars_dir):
sudo("mkdir -p %s" % env.galaxy_jars_dir)
_chown_galaxy(env, env.galaxy_jars_dir)
def _setup_install_dir(env):
"""Sets up install dir and ensures its owned by Galaxy"""
if not exists(env.galaxy_tools_dir):
sudo("mkdir -p %s" % env.galaxy_tools_dir)
_chown_galaxy(env, env.galaxy_tools_dir)
# Create a general-purpose ``bin`` directory under the galaxy_tools_dir
# and put it on the PATH so users can more easily add custom tools
bin_dir = os.path.join(env.galaxy_tools_dir, 'bin')
if not exists(bin_dir):
sudo("mkdir -p %s" % bin_dir)
_chown_galaxy(env, bin_dir)
line = "export PATH={0}:$PATH".format(bin_dir)
_add_to_profiles(line)
if not exists(env.galaxy_jars_dir):
sudo("mkdir -p %s" % env.galaxy_jars_dir)
_chown_galaxy(env, env.galaxy_jars_dir)
def _create_galaxy_db(env):
"""
Create a new PostgreSQL database for use by Galaxy
"""
c = _get_galaxy_db_configs(env)
if not exists(c["psql_data_dir"]):
sudo("mkdir -p {0}".format(c["psql_data_dir"]))
sudo("chown --recursive {0}:{0} {1}".format(c["psql_user"], c["psql_data_dir"]))
# Initialize a new database for Galaxy in ``psql_data_dir``
if _dir_is_empty(c["psql_data_dir"]):
sudo("{0} -D {1}".format(os.path.join(c["psql_bin_dir"], "initdb"), c["psql_data_dir"]), user=c["psql_user"])
# Set port for the database server
sed(c["psql_conf_file"], "#port = 5432", "port = {0}".format(c["psql_port"]), use_sudo=True)
sudo("chown {0}:{0} {1}".format(c["psql_user"], c["psql_conf_file"]))
# Start PostgreSQL server so a role for Galaxy user can be created
if not _postgres_running(env):
sudo(c["pg_start_cmd"], user=c["psql_user"])
started = True
else:
# Restart is required so port setting takes effect
sudo(
"{0} -D {1} -w -l {2} restart".format(c["pg_ctl_cmd"]),
c["psql_data_dir"],
c["psql_log"],
user=c["psql_user"],
)
started = False
# Create a role for env.galaxy_user
sudo('{0} -c"CREATE ROLE {1} LOGIN CREATEDB"'.format(c["psql_cmd"], env.galaxy_user), user=c["psql_user"])
# Create a Galaxy database
sudo(
"{0} -p {1} {2}".format(os.path.join(c["psql_bin_dir"], "createdb"), c["psql_port"], c["galaxy_db_name"]),
user=env.galaxy_user,
)
# Create a role for 'galaxyftp' user
sudo(
"{0} -c\"CREATE ROLE galaxyftp LOGIN PASSWORD '{1}'\"".format(c["psql_cmd"], c["galaxy_ftp_user_pwd"]),
user=c["psql_user"],
)
if started:
with settings(warn_only=True):
sudo("{0}".format(c["pg_stop_cmd"]), user=c["psql_user"])
exp = "export PATH={0}:$PATH".format(c["psql_bin_dir"])
_add_to_profiles(exp)
