def deploy(branch="master"):
"""
Pulls the latest changes from master, rebuilt and restarts the stack
"""
_check_env()
ENV.DEPLOYMENT_DATETIME = datetime.datetime.utcnow().isoformat()
deployment_branch = ENV.DEPLOYMENT_DATETIME.replace(":", "").replace(".", "")
lrun("git push origin {}".format(branch))
_copy_secrets()
with ENV.cd(ENV.project_dir):
docker_compose("run postgres backup before-deploy-at-{}.sqlc".format(ENV.DEPLOYMENT_DATETIME))
ENV.run("git fetch --all")
ENV.run("git checkout -f origin/{} -b {}".format(branch, deployment_branch))
_build_and_restart("django-a")
time.sleep(10)
# just to make sure they are on
docker_compose("start postgres")
docker_compose("start redis")
time.sleep(10)
_build_and_restart("django-b")
def site_config(path=None,
archive_name='latest_codalab_config.tar',
url=None,
module=None):
spath = 'src'
if path and os.path.exists(path):
path = os.path.abspath(path)
elif module:
mod = __import__(module)
if os.path.isdir(mod.__path__[0]):
path = mod.__path__[0]
else:
raise Exception("Must be a directory module")
with settings(warn_ony=True), lcd(path):
res = lrun('git diff --exit-code')
if res.return_code != 0:
raise Exception(
"*** Module has local changes. You must commit them.")
tmp = tempfile.mkdtemp()
fname = archive_name
tmpf = os.path.join(tmp, fname)
path = path.rstrip('/')
lrun('git archive --prefix=%s%s -o %s HEAD' %
(os.path.basename(path), os.path.sep, tmpf))
env.run('mkdir -p %s' % spath)
put(tmpf)
env.run('tar -C %s -xvf %s' % (spath, fname))
with virtualenv(env.venvpath):
env.run('pip install -U --force-reinstall ./%s' %
pathjoin(spath, os.path.basename(path)))
env.EXTERNAL_SITE_CONFIG = True
def _fetch_and_unpack(path, url, need_dir=True, wget_options=''):
tar_file, dir_name, tar_cmd = _get_expected_file(path, url)
if lexists(os.path.join(path, tar_file)):
lrun("rm -rf %s" % os.path.join(path, tar_file))
lrun("wget --no-check-certificate %s %s" % (wget_options, url))
vlrun("%s %s" % (tar_cmd, tar_file))
return _safe_dir_name(path, dir_name, need_dir)
def do_live():
lrun('cd %s && pip install -r requirements.txt' % LIVE_FOLDER)
lrun('cd %s && git checkout -- .' % LIVE_FOLDER)
lrun('cd %s && git pull github master' % LIVE_FOLDER)
lrun('cd %s && npm install' % LIVE_FOLDER)
lrun('cd %s && bower install' % LIVE_FOLDER)
do_no_network()
def site_config(path=None,archive_name='latest_codalab_config.tar',url=None,module=None):
spath = 'src'
if path and os.path.exists(path):
path = os.path.abspath(path)
elif module:
mod = __import__(module)
if os.path.isdir(mod.__path__[0]):
path = mod.__path__[0]
else:
raise Exception("Must be a directory module")
with settings(warn_ony=True),lcd(path):
res = lrun('git diff --exit-code')
if res.return_code != 0:
raise Exception("*** Module has local changes. You must commit them.")
tmp = tempfile.mkdtemp()
fname = archive_name
tmpf = os.path.join(tmp,fname)
path = path.rstrip('/')
lrun('git archive --prefix=%s%s -o %s HEAD' % (os.path.basename(path),os.path.sep,tmpf))
env.run('mkdir -p %s' % spath)
put(tmpf)
env.run('tar -C %s -xvf %s' % (spath,fname))
with virtualenv(env.venvpath):
env.run('pip install -U --force-reinstall ./%s' % pathjoin(spath,os.path.basename(path)))
env.EXTERNAL_SITE_CONFIG = True
def download_media(media_dir="/tmp", override="n"):
"""
Download and replace all files from media directory from remote environment to your local environment
"""
temp_dirpath = tempfile.mkdtemp()
media_dir = os.path.abspath(media_dir)
target_media_dir = os.path.join(media_dir, "media")
if override != "y" and prompt(
'{} directory will be override. Continue? [y/N]'.format(target_media_dir),
default='n',
validate=r'^y|n$'
) == 'n':
exit()
lrun('rm -rfv {}'.format(target_media_dir))
ENV.run("mkdir -p {}".format(temp_dirpath))
container_id = docker_compose("ps -q django-a").split()[0]
ENV.run(
"docker cp {}:/data/media/ {}".format(
container_id,
temp_dirpath
)
)
get("{}/media/".format(temp_dirpath), media_dir)
ENV.run("rm -rf {}".format(temp_dirpath))
def load_databases(domain,init):
f_domain = domain.replace('.au','').replace('.com','').replace('.net','').replace('-','')
f_domain = f_domain[:16] if len(f_domain) > 16 else f_domain
if init=='1':
#already exists on db server so no need to copy it over
f_sql_file = '/home/web_user/magento_skeleton_db_init.sql'
lrun('cp {0} /tmp/.'.format(f_sql_file))
f_sql_file = '/tmp/magento_skeleton_db_init.sql'
else:
sql_file = f_domain+'.sql'
lrun('mysqldump {0} > /tmp/{1}'.format(f_domain,sql_file))
f_sql_file = '/tmp/'+sql_file
#copy sql dump over to db server
if env.host_string == 'db1':
put(f_sql_file, '/tmp')
run("mysql --default-character-set=utf8 test_{0} < {1}".format(f_domain, f_sql_file))
run("mysql --default-character-set=utf8 {0} < {1}".format(f_domain, f_sql_file))
#remove uploaded dump sql file
#run('rm {0}'.format(f_sql_file))
#if init=='0':
# lrun('rm {0}'.format(f_sql_file))
# if prod update url
if env.host_string == 'db1':
update_url(domain)
def install_workflow():
"""Install Richard Bower CRUK-CI workflow manager
Checkout the workflow manager from repository.
"""
with lcd(env.lib_dir):
workflow_path = os.path.join(env.chipseq_installer, "workflow-manager")
lrun('cp -r %s .' % workflow_path)
def install_sicer():
"""Install SICER 1.1
"""
url = "http://home.gwu.edu/~wpeng/SICER_V1.1.tgz"
with lcd(env.tmp_dir):
dir_name = _fetch_and_unpack(env.tmp_dir, url)
with lcd(dir_name):
lrun("mv SICER %(sicer_dir)s" % env)
def test_connection():
"""Tries various simple fabric operations to test the connection to the server."""
run("echo testing echo...")
run("echo $LANG")
run("echo $LC_ALL")
sudo("echo testing sudo echo...")
lrun("touch /tmp/test-file")
put("/tmp/test-file", "/tmp/")
def install_git():
"""Install git 1.8.4.2
"""
url = "http://git-core.googlecode.com/files/git-1.8.4.2.tar.gz"
with lcd(env.tmp_dir):
dir_name = _fetch_and_unpack(env.tmp_dir, url)
with lcd(dir_name):
lrun("make prefix=%s all" % env.project_dir)
lrun("make prefix=%s install" % env.project_dir)
def test_connection():
"""Tries various simple fabric operations to test the connection to the server."""
run("echo testing echo...")
run("echo $LANG")
run("echo $LC_ALL")
sudo("echo testing sudo echo...")
lrun("touch /tmp/test-file")
put("/tmp/test-file", "/tmp/")
def setup_environment():
"""Copy adhoc environment variables, set CHIPSEQ_ROOT path and create tmp directory
"""
sed_chipseq_root = env.project_dir.replace('/', '\/')
setup_ori = os.path.join(env.chipseq_installer, env.env_setup)
setup_dest = os.path.join(env.project_dir, env.env_setup)
lrun("sed 's/\/Path\/To\/Edit\//%s/' %s > %s" %
(sed_chipseq_root, setup_ori, setup_dest))
_make_dir(env.tmp_dir)
def install_samtools():
"""Install samtools 0.1.18
"""
url = "http://sourceforge.net/projects/samtools/files/samtools/0.1.18/samtools-0.1.18.tar.bz2"
with lcd(env.tmp_dir):
dir_name = _fetch_and_unpack(env.tmp_dir, url)
with lcd(dir_name):
lrun("make")
# copy executables to bin
lrun("find . -perm /u=x -type f -exec cp {} %(bin_dir)s \;" % env)
def _install_rpy_lib():
"""Install RPy 1.0.3
"""
url = "http://sourceforge.net/projects/rpy/files/rpy/1.0.3/rpy-1.0.3.tar.gz"
with lcd(env.tmp_dir):
dir_name = _fetch_and_unpack(env.tmp_dir, url)
with lcd(dir_name):
lrun("sed -i 's/\[0\-9\]/\[0\-9\]\+/g' rpy_tools.py")
lrun("sed -i 's/Rdevices.h/Rembedded.h/g' src/RPy.h")
vlrun("python setup.py install")
def update(horizon_path=settings.HORIZON_ROOT):
"""Update the Front-end and its dependencies."""
# returns 1 if everything went OK, 0 otherwise
print 'Updating Horizon...'
with lcd(horizon_path):
lrun('git pull origin')
lrun('sudo python tools/install_venv.py')
print green('Horizon updated.')
return instance.run(horizon_path=horizon_path) #flag for the main task
def update(horizon_path=settings.HORIZON_ROOT):
"""Update the Front-end and its dependencies."""
# returns 1 if everything went OK, 0 otherwise
print 'Updating Horizon...'
with lcd(horizon_path):
lrun('git pull origin')
lrun('sudo python tools/install_venv.py')
print green('Horizon updated.')
return instance.run(horizon_path=horizon_path) #flag for the main task
def install_java():
"""Install Java 7
http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz
"""
tar_file = "jdk-7u51-linux-x64.tar.gz"
with lcd(env.tmp_dir):
lrun(
'wget --no-check-certificate --no-cookies --header "Cookie: oraclelicense=accept-securebackup-cookie" http://download.oracle.com/otn-pub/java/jdk/7u51-b13/%s -O %s'
% (tar_file, tar_file))
lrun("tar zxvf %s -C %s" % (tar_file, env.lib_dir))
def deploy():
"""
Pulls the latest changes from master, rebuilt and restarts the stack
"""
lrun("git push origin master")
copy_secrets()
with env.cd(env.project_dir):
env.run("git pull origin master")
build()
restart()
def set_up_as_service(absolute_horizon_path=None):
if not absolute_horizon_path:
absolute_horizon_path = os.getcwd() + '/' + settings.HORIZON_ROOT
in_file = open('conf/horizon_idm.conf')
src = string.Template(in_file.read())
out_file = open("tmp_horizon_idm.conf", "w")
out_file.write(src.substitute({
'absolute_horizon_path': absolute_horizon_path}))
out_file.close()
lrun('sudo cp tmp_horizon_idm.conf /etc/init/horizon_idm.conf')
lrun('sudo rm tmp_horizon_idm.conf')
def deploy():
"""
Pulls the latest changes from master, rebuilt and restarts the stack
"""
lrun("git push origin master")
copy_secrets()
with env.cd(env.project_dir):
env.run("git pull origin master")
build()
restart()
def set_up_as_service(absolute_keystone_path=None):
if not absolute_keystone_path:
absolute_keystone_path = os.getcwd() + '/' + settings.KEYSTONE_ROOT
in_file = open('conf/keystone_idm.conf')
src = string.Template(in_file.read())
out_file = open("tmp_keystone_idm.conf", "w")
out_file.write(
src.substitute({'absolute_keystone_path': absolute_keystone_path}))
out_file.close()
lrun('sudo cp tmp_keystone_idm.conf /etc/init/keystone_idm.conf')
lrun('sudo rm tmp_keystone_idm.conf')
def install_r():
"""Install R 2.15.0
"""
_make_dir(env.r_lib_dir)
url = "http://cran.r-project.org/src/base/R-2/R-2.15.0.tar.gz"
option = "--enable-R-shlib"
if not lexists(os.path.join(env.r_dir, "bin/R")):
_get_install(url, env, _configure_make, option)
# create symlinks in bin for installation on mac only
if not lexists(os.path.join(env.bin_dir, "R")):
lrun('ln -fs %(r_dir)s/bin/R %(bin_dir)s/R' % env)
lrun('ln -fs %(r_dir)s/bin/Rscript %(bin_dir)s/Rscript' % env)
def install_picard():
"""Install Picard 1.96
"""
version = "1.96"
url = 'http://downloads.sourceforge.net/project/picard/picard-tools/%s/picard-tools-%s.zip' % (
version, version)
picard_dir = os.path.join(env.bin_dir, "picard")
_make_dir(picard_dir)
with lcd(env.tmp_dir):
dir_name = _fetch_and_unpack(env.tmp_dir, url)
with lcd(dir_name):
lrun("mv *.jar %s" % picard_dir)
def database_create(keystone_path=settings.KEYSTONE_ROOT, verbose=True):
add_verbose = '-v' if verbose else ''
with lcd(keystone_path):
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync').format(v=add_verbose))
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync --extension endpoint_filter').format(v=add_verbose))
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync --extension=oauth2').format(v=add_verbose))
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync --extension=roles').format(v=add_verbose))
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync --extension=user_registration').format(v=add_verbose))
def database_create(keystone_path=settings.KEYSTONE_ROOT, verbose=True):
add_verbose = '-v' if verbose else ''
with lcd(keystone_path):
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync').format(v=add_verbose))
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync --extension endpoint_filter').format(v=add_verbose))
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync --extension=oauth2').format(v=add_verbose))
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync --extension=roles').format(v=add_verbose))
lrun(('sudo tools/with_venv.sh bin/keystone-manage {v}'
' db_sync --extension=user_registration').format(v=add_verbose))
def install(horizon_path=settings.HORIZON_ROOT, version=None):
"""Download and install the Front-end and its dependencies."""
if os.path.isdir(horizon_path[:-1]):
print 'Already downloaded.'
else:
lrun('git clone https://github.com/ging/horizon.git \
{0}'.format(horizon_path))
with lcd(horizon_path):
if not version:
version = settings.KEYROCK_VERSION
lrun('git checkout tags/keyrock-{0}'.format(version))
dependencies = ' '.join(settings.UBUNTU_DEPENDENCIES['horizon'])
lrun('sudo apt-get install -y {0}'.format(dependencies))
lrun('sudo python tools/install_venv.py')
path = horizon_path + '/openstack_dashboard/local/'
class Template(string.Template):
delimiter = '$$'
template_settings = Template(open(path + 'local_settings.py.example').read())
out_file = open(path + "local_settings.py", "w")
out_file.write(
template_settings.substitute({
'IDM_NAME': settings.IDM_USER_CREDENTIALS['username'],
'IDM_PASS': settings.IDM_USER_CREDENTIALS['password'],
'IDM_PROJECT': settings.IDM_USER_CREDENTIALS['project'],
'KEYSTONE_ADDRESS': settings.KEYSTONE_INTERNAL_ADDRESS,
'KEYSTONE_PUBLIC_PORT':settings.KEYSTONE_PUBLIC_PORT,
}))
out_file.close()
instance.run(horizon_path=horizon_path) # run check task
def install_openssl():
"""Install openssl 1.0.1e
For UCSC tools that gives libssl.so.10 error while loading shared libraries
"""
url = "http://www.openssl.org/source/openssl-1.0.1e.tar.gz"
with lcd(env.tmp_dir):
dir_name = _fetch_and_unpack(env.tmp_dir, url)
with lcd(dir_name):
lrun("./config --prefix=%s --shared" % env.project_dir)
lrun("make")
lrun("make install")
with lcd(env.lib_dir):
lrun("ln -s ../lib64/libssl.so.1.0.0 libssl.so.10")
lrun("ln -s ../lib64/libcrypto.so.1.0.0 libcrypto.so.10")
def install_macs():
"""Install MACS 1.4.2
Model-based Analysis for ChIP-Seq.
http://liulab.dfci.harvard.edu/MACS/
"""
version = "1.4.2"
url = "https://github.com/downloads/taoliu/MACS/MACS-%s.tar.gz" % version
with lcd(env.tmp_dir):
dir_name = _fetch_and_unpack(env.tmp_dir, url)
with lcd(dir_name):
vlrun("python setup.py install")
lrun("chmod a+rwx bin/*")
lrun("find bin/. -perm /u=x -type f -exec cp {} %(bin_dir)s \;" %
env)
def install_ucsc_tools():
"""Install useful executables from UCSC.
see https://github.com/chapmanb/cloudbiolinux/blob/master/cloudbio/custom/bio_nextgen.py
for an up-to-date version
"""
tools = [
"liftOver", "faToTwoBit", "twoBitToFa", "bedToBigBed", "wigToBigWig",
"bedGraphToBigWig"
]
url = "http://hgdownload.cse.ucsc.edu/admin/exe/linux.x86_64/"
for tool in tools:
with lcd(env.bin_dir):
if not lexists(os.path.join(env.bin_dir, tool)):
lrun("wget %s%s" % (url, tool))
lrun("chmod a+rwx %s" % tool)
def update(keystone_path=settings.KEYSTONE_ROOT):
"""Update the Back-end and its dependencies."""
# returns 1 if everything went OK, 0 otherwise
print 'Updating Keystone...'
with lcd(keystone_path):
lrun('git pull origin')
lrun('sudo python tools/install_venv.py')
print 'Syncing database...'
database_create(keystone_path, True)
print green('Keystone updated.')
if not check(keystone_path):
return 0 # flag for the main task
else:
return 1 # flag for the main task
def configure_meme():
with lcd(env.annotation_dir):
URLForJasparAll = "http://jaspar.genereg.net/html/DOWNLOAD/ARCHIVE/JASPAR2010/JASPAR_CORE/non_redundant/all_species/FlatFileDir/"
lrun(
'wget -r -nH --cut-dirs=2 --no-parent --reject=\"index.html*\" %s '
% (URLForJasparAll))
JasparLocation = os.path.join(
env.annotation_dir,
"ARCHIVE/JASPAR2010/JASPAR_CORE/non_redundant/all_species/FlatFileDir/"
)
MemeJasparLocation = os.path.join(
env.annotation_dir,
"ARCHIVE/JASPAR2010/JASPAR_CORE/Jaspar_NonRedunadant.meme")
ConvertCMD = os.path.join(env.bin_dir, "meme/bin/jaspar2meme -pfm")
lrun("%s %s > %s" % (ConvertCMD, JasparLocation, MemeJasparLocation))
def update(keystone_path=settings.KEYSTONE_ROOT):
"""Update the Back-end and its dependencies."""
# returns 1 if everything went OK, 0 otherwise
print 'Updating Keystone...'
with lcd(keystone_path):
lrun('git pull origin')
lrun('sudo python tools/install_venv.py')
print 'Syncing database...'
database_create(keystone_path, True)
print green('Keystone updated.')
if not check(keystone_path):
return 0 # flag for the main task
else:
return 1 # flag for the main task
def env(name="prod"):
"""
Set environment based on your local .env.<name> file
"""
filename = ".env.{}".format(name)
if not os.path.isfile(filename):
print(red("Missing {} file".format(filename)))
raise SystemExit()
with open(".env.{}".format(name)) as env_file:
ENV.name = name
for line in env_file:
line = line.strip()
if not line or line.startswith("#"):
continue
key, value = line.split("=")
if key.startswith("FAB_") and value:
ENV.__setattr__(key.replace("FAB_", "").lower(), value)
if ENV.name == "local":
ENV.run = lambda *args, **kwargs: lrun(capture=True, *args, **kwargs)
ENV.cd = lcd
else:
ENV.run = run # if you don't log in as root, replace with 'ENV.run = sudo'
ENV.cd = cd
def vlrun(command):
"""Run a command in a virtual environment. This prefixes the run command with the source command.
Usage:
vlrun('pip install tables')
"""
source = 'source %(project_dir)s/bin/%(activate)s && source %(project_dir)s/%(env_setup)s && ' % env
return lrun(source + command, shell='%s' % env.shell)
def _runwrapper(*args, **kwargs):
"""when attempting to run a command as root on the dev machine, it runs the
command as a sudo otherwise it will try to ssh in as the root user."""
_sudo = lambda cmd, *args, **kwargs: lrun("sudo %s" % cmd, *args, **kwargs)
if env.user == 'root': # or (kwargs.has_key('use_sudo') and kwargs['use_sudo'] == True):
# ick. no local equivalent of `sudo`
cmd = args[0]
return _sudo(cmd, *args[1:], **kwargs)
#cmd = 'sudo %s' % cmd
#if kwargs.has_key('use_sudo'):
# del kwargs['use_sudo']
#return lrun(cmd, *args[1:], **kwargs)
elif kwargs.has_key('use_sudo') and kwargs['use_sudo'] == True:
del kwargs['use_sudo']
cmd = args[0]
return _sudo(cmd, *args[1:], **kwargs)
if kwargs.has_key('use_sudo'):
del kwargs['use_sudo']
return lrun(*args, **kwargs)
def deploy():
"""
Pulls the latest changes from master, rebuilt and restarts the stack
"""
lrun("git push origin master")
copy_secrets()
with env.cd(env.project_dir):
docker_compose("run postgres backup")
env.run("git pull origin master")
build_and_restart("django")
# just to make sure they are on
docker_compose("start postgres")
docker_compose("start redis")
time.sleep(10)
build_and_restart("django-failover")
def deploy():
"""
Pulls the latest changes from master, rebuilt and restarts the stack
"""
lrun("git push origin master")
copy_secrets()
with env.cd(env.project_dir):
docker_compose("run postgres backup")
env.run("git pull origin master")
build_and_restart("django")
# just to make sure they are on
docker_compose("start postgres")
docker_compose("start redis")
time.sleep(10)
build_and_restart("django-failover")
def install_chipseq_pipeline():
"""Get the latest chipseq code from github.
"""
with lcd(env.project_dir):
lrun(
"wget --no-check-certificate -r https://github.com/crukci-bioinformatics/chipseq-pipeline/archive/master.zip -O master-pipeline.zip"
)
lrun("unzip master-pipeline.zip")
with lcd(env.chipseq_path):
lrun(
"( ( echo '#!/usr/bin/env Rscript' ; echo 'RLIBSVar = \"%s\"' ; sed '1,2d' RScripts/Kick.r ) > RScripts/ChipSeq.r )"
% env.r_lib_dir)
lrun("chmod a+x RScripts/ChipSeq.r")
def db_clone(source, options_table):
source = getattr(config, source)
mysqldump = 'mysqldump --add-drop-table -h %s -u %s -p%s %s%s' % (
source['wordpressConfig']['DB_HOST'],
source['wordpressConfig']['DB_USER'],
source['wordpressConfig']['DB_PASSWORD'] ,
source['wordpressConfig']['DB_NAME'],
' --ignore-table=%s.wp_options' % source['wordpressConfig']['DB_NAME'] if options_table is 'no' else '')
mysql_login = '******' % (
env.target['wordpressConfig']['DB_HOST'],
env.target['wordpressConfig']['DB_USER'],
env.target['wordpressConfig']['DB_PASSWORD'],
env.target['wordpressConfig']['DB_NAME'])
with quiet():
if source == config.local:
lrun("%s | ssh %s '%s'" %
(mysqldump,
env.target['host'],
mysql_login))
else:
run("ssh %s '%s' | %s" %
(source['host'],
mysqldump,
mysql_login))
def set_up_as_service(absolute_keystone_path=None):
if not absolute_keystone_path:
absolute_keystone_path = os.getcwd() + '/' + settings.KEYSTONE_ROOT
in_file = open('conf/keystone_idm.conf')
src = string.Template(in_file.read())
out_file = open("tmp_keystone_idm.conf", "w")
out_file.write(src.substitute({
'absolute_keystone_path': absolute_keystone_path}))
out_file.close()
lrun('sudo cp tmp_keystone_idm.conf /etc/init/keystone_idm.conf')
lrun('sudo rm tmp_keystone_idm.conf')
lrun('sudo ln -sf /etc/init/keystone_idm.conf /etc/init.d/keystone_idm')
def install_bedtools():
"""Install BEDTools 2.17.0
"""
url = "http://bedtools.googlecode.com/files/BEDTools.v2.17.0.tar.gz"
with lcd(env.tmp_dir):
# cannot _fetch_and_unpack return because package name does not match unpacked dir
_fetch_and_unpack(env.tmp_dir, url, False)
with lcd("bedtools-2.17.0"):
lrun("make clean")
lrun("make all")
lrun("find bin/. -perm /u=x -type f -exec cp {} %(bin_dir)s \;" %
env)
def install(horizon_path=settings.HORIZON_ROOT, version=None, unattended=False):
"""Download and install the Front-end and its dependencies."""
if os.path.isdir(horizon_path[:-1]):
print "Already downloaded."
else:
lrun(
"git clone https://github.com/ging/horizon.git \
{0}".format(
horizon_path
)
)
with lcd(horizon_path):
if not version:
version = settings.KEYROCK_VERSION
lrun("git checkout tags/keyrock-{0}".format(version))
dependencies = " ".join(settings.UBUNTU_DEPENDENCIES["horizon"])
lrun("sudo apt-get install -y {0}".format(dependencies))
lrun("sudo python tools/install_venv.py")
path = horizon_path + "/openstack_dashboard/local/"
class Template(string.Template):
delimiter = "$$"
template_settings = Template(open(path + "local_settings.py.example").read())
out_file = open(path + "local_settings.py", "w")
out_file.write(
template_settings.substitute(
{
"IDM_NAME": settings.IDM_USER_CREDENTIALS["username"],
"IDM_PASS": settings.IDM_USER_CREDENTIALS["password"],
"IDM_PROJECT": settings.IDM_USER_CREDENTIALS["project"],
"KEYSTONE_ADDRESS": settings.KEYSTONE_INTERNAL_ADDRESS,
"KEYSTONE_PUBLIC_PORT": settings.KEYSTONE_PUBLIC_PORT,
}
)
)
out_file.close()
instance.run(horizon_path=horizon_path, unattended=unattended) # run check task
def install_meme():
"""Install meme 4.9.1
"""
url = "http://ebi.edu.au/ftp/software/MEME/4.9.1/meme_4.9.1.tar.gz"
with lcd(env.tmp_dir):
dir_name = _fetch_and_unpack(env.tmp_dir, url)
with lcd(dir_name):
lrun(
"./configure --prefix=%(meme_dir)s --with-url='http://meme.nbcr.net/meme' --with-perl=%(bin_dir)s/perl/bin/perl --with-python=%(bin_dir)s/python2.7"
% env)
lrun("make")
lrun("make install")
def update(horizon_path=settings.HORIZON_ROOT, version=None):
"""Update the Front-end and its dependencies."""
# returns 1 if everything went OK, 0 otherwise
print "Updating Horizon..."
with lcd(horizon_path):
lrun("git fetch")
if not version:
version = settings.KEYROCK_VERSION
lrun("git checkout tags/keyrock-{0}".format(version))
lrun("sudo python tools/install_venv.py")
print green("Horizon updated.")
return instance.run(horizon_path=horizon_path) # flag for the main task
def install(keystone_path=settings.KEYSTONE_ROOT):
"""Download and install the Back-end and its dependencies."""
if os.path.isdir(keystone_path[:-1]):
print 'Already downloaded.'
else:
lrun(('git clone https://github.com/ging/keystone.git '
'{0}').format(keystone_path))
with lcd(keystone_path):
dependencies = ' '.join(settings.UBUNTU_DEPENDENCIES['keystone'])
lrun('sudo apt-get install -y {0}'.format(dependencies))
lrun('sudo cp etc/keystone.conf.sample etc/keystone.conf')
lrun('sudo python tools/install_venv.py')
# Uncomment config file
with lcd('etc/'):
lrun(("sudo sed -i "
"'s/#admin_token=ADMIN/admin_token={0}/g' "
"keystone.conf").format(settings.KEYSTONE_ADMIN_TOKEN))
lrun(("sudo sed -i "
"'s/#admin_port=35357/admin_port={0}/g' "
"keystone.conf").format(settings.KEYSTONE_ADMIN_PORT))
lrun(("sudo sed -i "
"'s/#public_port=5000/public_port={0}/g' "
"keystone.conf").format(settings.KEYSTONE_PUBLIC_PORT))
print 'Done!'
check(keystone_path) # run check
def build(env, version):
lrun("jekyll build")
path = get_deploy_path(env, version)
run("mkdir -p {}".format(path))
put("_site/*", path)
def do_dev():
lrun('cd %s && git checkout develop' % DEV_FOLDER)
lrun('cd %s && git pull github develop' % DEV_FOLDER)
lrun('cd %s && git push github develop' % DEV_FOLDER)
lrun('cd %s && pip install -r requirements.txt' % DEV_FOLDER)
lrun('cd %s && git checkout master && git merge develop' % DEV_FOLDER)
lrun('cd %s && git push github master' % DEV_FOLDER)
lrun('cd %s && git checkout develop' % DEV_FOLDER)
def _install_dependencies():
command = settings.UBUNTU_DEPENDENCIES['install_command']
dependencies = ' '.join(settings.UBUNTU_DEPENDENCIES['dependencies'])
lrun('{command} {dependencies}'.format(command=command,
dependencies=dependencies))
print 'Dependencies correctly installed'
def start():
"""Runs the service."""
lrun('sudo service keystone_idm start')
def database_delete(keystone_path=settings.KEYSTONE_ROOT):
db_path = keystone_path + settings.KEYSTONE_DEV_DATABASE
if os.path.isfile(db_path):
lrun('sudo rm ' + db_path)
def sync(horizon_path=settings.HORIZON_ROOT):
with lcd(horizon_path):
lrun(('sudo tools/with_venv.sh python access_control_xacml.py '))
def dev_server(address=settings.HORIZON_DEV_ADDRESS,
horizon_path=settings.HORIZON_ROOT):
"""Run horizon server for development purposes"""
with lcd(horizon_path):
lrun(('sudo tools/with_venv.sh python manage.py runserver '
'{0}').format(address))
def stop():
"""Stops the service."""
lrun('sudo service keystone_idm stop')
def _gulp_cmd(cmd):
lrun('gulp --gulpfile={} {}'.format(gulp_file, cmd))
def restart():
"""Restarts the service."""
lrun('sudo service keystone_idm restart')
def install():
with lcd(theme_path):
lrun('npm install && bower install')
def dev_server(keystone_path=settings.KEYSTONE_ROOT):
"""Runs the server in dev mode."""
with lcd(keystone_path):
lrun('sudo tools/with_venv.sh bin/keystone-all -v')
def config(directory):
env.user = '******'
lrun('test -d {0}'.format(directory))
run('mkdir -p {0}'.format(SETTINGS_DIR))
rsync_project(local_dir='{0}/'.format(directory), remote_dir=SETTINGS_DIR)
run('chown -R root: {0}'.format(SETTINGS_DIR))
def checkURLs(domain):
result = lrun('linkchecker --no-status -a -ohtml http://{0}'.format(domain))
