def check_brew_cellar():
"""
Find the brewing cellar (Mac OSX)
"""
with hide('output'):
# This yields something like "Homebrew 1.4.2-14-g3e99504"
# but might be different in earlier versions (or newer)
# so we do what we can to clean it up
version = run('brew --version')
if 'Homebrew ' in version:
version = version.split()[1].strip()
if '-' in version:
version = version.split('-')[0].strip()
version = tuple(map(int, version.split('.')))
# I'm not sure exactly when --cellar was introduced, but it was already
# there in 0.9.9. On the other hand HOMEBREW_CELLAR was also still
# outputted via brew config in 0.9.9, so I think it's safe to make the
# cut at 1.0.
if version >= (1, 0):
cellar = run('brew --cellar')
else:
cellar = run('brew config | grep HOMEBREW_CELLAR').split(":")[1].strip()
return cellar
def create_user(user):
"""
Creates a user in the system.
"""
# TODO: Check if the user exists
# Also, these commands are linux-specific,
# there are others that work on MacOS
group = user.lower()
sudo('groupadd ' + group, warn_only=True)
sudo('useradd -g {0} -m -s /bin/bash {1}'.format(group, user),
warn_only=True)
sudo('mkdir /home/{0}/.ssh'.format(user), warn_only=True)
sudo('chmod 700 /home/{0}/.ssh'.format(user))
sudo('chown -R {0}:{1} /home/{0}/.ssh'.format(user, group))
# Copy the public key of our SSH key if we're using one
public_key = get_fab_public_key()
if public_key:
sudo("echo '{0}' >> /home/{1}/.ssh/authorized_keys".format(
public_key, user))
sudo('chmod 600 /home/{0}/.ssh/authorized_keys'.format(user))
sudo('chown {0}:{1} /home/{0}/.ssh/authorized_keys'.format(
user, group))
# openSUSE creates a suboptimal ~/.profile because it shows an error message
# if /etc/profile doesn't exist (which is the case on openSUES dockers),
# so we comment out that particular line
if get_linux_flavor() == 'openSUSE':
with settings(user=user):
run('''sed -i 's/^test -z "$PROFILEREAD".*/#\\0/' ~/.profile ''')
def start_unicorn():
"""
Starts the gunicorn daemon which in turn will be called by nginx.
"""
HOME = home()
env.APP_INSTALL_DIR = os.path.abspath(os.path.join(HOME, APP_INSTALL_DIR_NAME))
env.APP_ROOT_DIR = os.path.abspath(os.path.join(HOME, APP_ROOT_DIR_NAME))
env.APP_SRC_DIR = os.path.abspath(os.path.join(HOME, APP_SRC_DIR_NAME))
NAME = "RASVAMT"
FLASKDIR = env.APP_INSTALL_DIR+'/src'
SOCKFILE = env.APP_INSTALL_DIR+'/sock'
PIDFILE = env.APP_INSTALL_DIR+'/gunicorn.pid'
USER = env.APP_USER
GROUP = env.APP_USER
NUM_WORKERS = '3'
with settings(user=env.APP_USER):
with cd(FLASKDIR):
run('{0}/bin/gunicorn main:app -b 127.0.0.1:5000 -D '.format(env.APP_INSTALL_DIR) +\
' --name {0}'.format(NAME) + \
' --user={0} --group={1}'.format(USER, GROUP) +\
' --bind=unix:{0}'.format(SOCKFILE) +\
' --workers={0}'.format(NUM_WORKERS) +\
' --log-level=debug --pythonpath={0}'.format(FLASKDIR) +\
' -p {0}'.format(PIDFILE)
)
def prepare_APP_data_dir():
"""Creates a new APP root directory"""
"""
Install the content of the NGAS portal
"""
if not exists('{0}/ngas/import/'.format(APP_install_dir())):
run('mkdir {0}/ngas/import/'.format(APP_install_dir()), warn_only=True)
put('data/*.zexp', '{0}/ngas/import/'.format(APP_install_dir()))
if not exists('{0}/../NGAS'.format(APP_install_dir())):
run('mkdir NGAS', warn_only=True)
if not exists('{0}/../NGAS/ngas.sqlite'.format(APP_install_dir())):
put('data/ngas.sqlite', '{0}/../NGAS/'.format(APP_install_dir()))
def download(url, target=None, root=False):
if target is None:
parts = urlparse.urlparse(url)
target = parts.path.split('/')[-1]
if check_command('wget'):
cmd = 'wget --no-check-certificate -q -O {0} {1}'.format(target, url)
elif check_command('curl'):
cmd = 'curl -o {0} {1}'.format(target, url)
else:
raise Exception("Neither wget nor curl are installed")
if root:
sudo(cmd)
else:
run(cmd)
return target
def APP_build_cmd():
env.APP_INSTALL_DIR = os.path.abspath(os.path.join(home(), APP_INSTALL_DIR_NAME))
env.APP_ROOT_DIR = os.path.abspath(os.path.join(home(), APP_ROOT_DIR_NAME))
env.APP_SRC_DIR = os.path.abspath(os.path.join(home(), APP_SRC_DIR_NAME))
build_cmd = []
# create RASVAMT directories and chown to correct user and group
run('mkdir -p {0}'.format(env.APP_INSTALL_DIR))
# This not working for some reason
# sudo('chown -R {0}:{1} {2}'.format(env.USERS[0], GROUP, APP_INSTALL_DIR))
run('ln -s {0}/src {1}/src'.format(env.APP_SRC_DIR, env.APP_INSTALL_DIR))
return ' '.join(build_cmd)
def check_dir(directory):
"""
Check existence of remote directory
"""
res = run(
"""if [ -d {0} ]; then echo 1; else echo ; fi""".format(directory))
return res
def prepare_ngas_data_dir():
"""Creates a new NGAS root directory"""
info('Preparing NGAS root directory')
nrd = APP_root_dir()
tgt_cfg = os.path.join(nrd, 'cfg', 'ngamsServer.conf')
with cd(APP_source_dir()):
cmd = ['./prepare_ngas_root.sh']
if 'NGAS_OVERWRITE_ROOT' in env and env.NGAS_OVERWRITE_ROOT:
cmd.append('-f')
cmd.append(nrd)
res = run(' '.join(cmd), quiet=True)
if res.succeeded:
success("NGAS data directory ready")
env.tgt_cfg = tgt_cfg
return tgt_cfg
# Deal with the errors here
error = 'NGAS root directory preparation under {0} failed.\n'.format(nrd)
if res.return_code == 2:
error = (nrd +
" already exists. Specify NGAS_OVERWRITE_ROOT to overwrite, "
"or a different NGAS_ROOT_DIR location")
else:
error = res
abort(error)
def check_command(command, *args, **kwargs):
"""
Check existence of command remotely
"""
res = run(
'if command -v {0} &> /dev/null ;then command -v {0};else echo ;fi'.
format(command), *args, **kwargs)
return res
def install_user_profile():
"""
Put the activation of the virtualenv into the login profile of the user
unless the APP_DONT_MODIFY_BASHPROFILE environment variable is defined
NOTE: This will be executed for the user running APP.
"""
if run('echo $APP_DONT_MODIFY_BASHPROFILE') or \
'APP_NO_BASH_PROFILE' in env:
return
nid = APP_install_dir()
nrd = APP_root_dir()
with cd("~"):
if not exists(".bash_profile_orig"):
run('cp .bash_profile .bash_profile_orig', warn_only=True)
else:
run('cp .bash_profile_orig .bash_profile')
script = ('if [ -f "{0}/bin/activate" ]'.format(nid),
'then',
' source "{0}/bin/activate"'.format(nid),
'fi',
'export APP_PREFIX="{0}"'.format(nrd))
run("echo '{0}' >> .bash_profile".format('\n'.join(script)))
success("~/.bash_profile edited for automatic virtualenv sourcing")
def install_homebrew():
"""
Task to install homebrew on Mac OSX.
NOTE: This should not be done if macports is installed already.
"""
lf = get_linux_flavor()
if lf != 'Darwin':
puts(red("Potentially this is not a Mac OSX installation: {0}".format(lf)))
raise(ValueError)
if check_command('port'):
puts(red('MacPorts is installed and it is not recommended to mix it with homebrew!!'))
puts(red('Bailing out!'))
raise(ValueError)
return
if not check_command('brew'):
run('ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"')
else:
puts(red('Homebrew is installed already! New installation not required.'))
def check_user(user):
"""
Task checking existence of user
"""
res = run(
'if id -u "{0}" >/dev/null 2>&1; then echo 1; else echo 0; fi;'.format(
user))
if res == '0':
puts('User {0} does not exist'.format(user))
return False
else:
return True
def cleanup_container():
# Clean downloaded packages, remove unnecessary packages
#
# This is obviously a CentOS 7 hardcoded list, but we already hardcode
# CentOS 7 as the FROM image in our build file so we are basically building
# up on that assumption. Generalising all this logic would require quite
# some effort. but since it is not necessarily something we need or want, it
# is kind of ok to live with this sin.
for pkg in ('autoconf', 'bzip2-devel', 'cpp', 'groff-base', 'krb5-devel',
'less', 'libcom_err-devel', 'libgnome-keyring', 'libedit',
'libgomp', 'libkadm5', 'libselinux-devel', 'm4', 'mpfr',
'pcre-devel', 'rsync', 'libverto-devel', 'libmpc', 'gcc',
'gdbm-devel', 'git', 'glibc-devel', 'glibc-headers',
'kernel-headers', 'libdb-devel', 'make', 'openssl-devel',
'patch', 'perl', 'postgresql', 'postgresql-libs',
'python-devel', 'readline-devel', 'sqlite-devel', 'sudo',
'wget', 'zlib-devel', 'libffi-devel'):
run('yum --assumeyes --quiet remove %s' % (pkg, ), warn_only=True)
run('yum clean all')
# Remove user directories that are not needed anymore
with settings(user=APP_user()):
# By default we do not ship the image with a working APP directory
to_remove = ['~/.cache']
if not docker_keep_APP_src():
to_remove.append(APP_source_dir())
if not docker_keep_APP_root():
to_remove.append(APP_root_dir())
for d in to_remove:
run('rm -rf %s' % d, )
def virtualenv_setup():
"""
Creates a new virtualenv that will hold the APP installation
"""
APPInstallDir = APP_install_dir()
if check_dir(APPInstallDir):
overwrite = APP_overwrite_installation()
if not overwrite:
msg = ("%s exists already. Specify APP_OVERWRITE_INSTALLATION "
"to overwrite, or a different APP_INSTALL_DIR location")
abort(msg % (APPInstallDir,))
run("rm -rf %s" % (APPInstallDir,))
# Check which python will be bound to the virtualenv
ppath = check_python()
if not ppath:
ppath = python_setup(os.path.join(home(), 'python'))
# Use our create_venv.sh script to create the virtualenv
# It already handles the download automatically if no virtualenv command is
# found in the system, and also allows to specify a python executable path
script_path = os.path.dirname(os.path.realpath(__file__))+'/create_venv.sh'
put(script_path, APP_source_dir()+'/create_venv.sh')
with cd(APP_source_dir()):
run("/bin/bash create_venv.sh -p {0} {1}".format(ppath, APPInstallDir))
# Download this particular certifcate; otherwise pip complains
# in some platforms
if APP_use_custom_pip_cert():
if not(check_dir('~/.pip')):
run('mkdir ~/.pip')
with cd('~/.pip'):
download('http://curl.haxx.se/ca/cacert.pem')
run('echo "[global]" > ~/.pip/pip.conf; echo '
'"cert = {0}/.pip/cacert.pem" >> ~/.pip/pip.conf;'.format(home()))
# Update pip and install wheel; this way we can install binary wheels from
# PyPI if available (like astropy)
# TODO: setuptools and python-daemon are here only because
# python-daemon 2.1.2 is having a problem to install via setuptools
# but not via pip (see https://pagure.io/python-daemon/issue/2 and
# https://pagure.io/python-daemon/issue/3).
# When this problem is fixed we'll fix our dependency on python-daemo
# to avoid this issue entirely
virtualenv('pip install -U pip wheel setuptools python-daemon')
success("Virtualenv setup completed")
def prepare_APP_data_dir():
"""Creates a new APP root directory"""
info('Preparing {0} root directory'.format(env.APP_NAME))
nrd = APP_root_dir()
# tgt_cfg = os.path.join(nrd, 'cfg', 'ngamsServer.conf')
tgt_cfg = None
res = run('mkdir -p {0}'.format(nrd))
with cd(APP_source_dir()):
for d in env.APP_DATAFILES:
res = run('scp -r {0} {1}/.'.format(d, nrd), quiet=True)
if res.succeeded:
success("{0} data directory ready".format(env.APP_NAME))
return tgt_cfg
# Deal with the errors here
error = '{0} root directory preparation under {1} failed.\n'.format(
env.APP_NAME, nrd)
if res.return_code == 2:
error = (nrd + " already exists. Specify APP_OVERWRITE_ROOT to "
"overwrite, or a different APP_ROOT_DIR location")
else:
error = res
abort(error)
def python_setup(ppath):
"""
Ensure that there is the right version of python available
If not install it from scratch in user directory.
"""
with cd('/tmp'):
download(env.APP_PYTHON_URL)
base = os.path.basename(env.APP_PYTHON_URL)
pdir = os.path.splitext(base)[0]
run('tar -xzf {0}'.format(base))
with cd('/tmp/{0}'.format(pdir)):
puts('Python BUILD log-file can be found in: /tmp/py_install.log')
puts(green('Configuring Python.....'))
run('./configure --prefix {0} > /tmp/py_install.log 2>&1;'.format(
ppath))
puts(green('Building Python.....'))
run('make >> /tmp/py_install.log 2>&1;')
puts(green('Installing Python.....'))
run('make install >> /tmp/py_install.log 2>&1')
ppath = '{0}/bin/python{1}'.format(ppath, env.APP_PYTHON_VERSION)
return ppath
def APP_build_cmd():
# The installation of the bsddb package (needed by ngamsCore) is in
# particular difficult because it requires some flags to be passed on
# (particularly if using MacOSX's port
build_cmd = []
linux_flavor = get_linux_flavor()
if linux_flavor == 'Darwin':
pkgmgr = check_brew_port()
if pkgmgr == 'brew':
cellardir = check_brew_cellar()
db_version = run(
'ls -tr1 {0}/berkeley-db@4'.format(cellardir)).split()[-1]
db_dir = '{0}/berkeley-db@4/{1}'.format(cellardir, db_version)
build_cmd.append('BERKELEYDB_DIR={0}'.format(db_dir))
if not settings['NGAS_NO_CLIENT']:
build_cmd.append('CFLAGS=-I{0}/include'.format(db_dir))
build_cmd.append('LDFLAGS=-L{0}/lib'.format(db_dir))
else:
incdir = MACPORT_DIR + '/include/db60'
libdir = MACPORT_DIR + '/lib/db60'
build_cmd.append('BERKELEYDB_INCDIR=' + incdir)
build_cmd.append('BERKELEYDB_LIBDIR=' + libdir)
if not settings['NGAS_NO_CLIENT']:
build_cmd.append('CFLAGS=-I' + incdir)
build_cmd.append('LDFLAGS=-L' + libdir)
build_cmd.append(
'YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1')
if settings['NGAS_NO_CRC32C']:
build_cmd.append('NGAS_NO_CRC32C=1')
build_cmd.append('./build.sh')
if not settings['NGAS_NO_CLIENT']:
build_cmd.append("-c")
if settings['NGAS_DEVELOP']:
build_cmd.append("-d")
if not settings['NGAS_DOC_DEPENDENCIES']:
build_cmd.append('-D')
return ' '.join(build_cmd)
def copy_sources():
"""
Creates a copy of the APP sources in the target host.
"""
# We don't open all the git repositories to the world, so for the time
# being we always make a tarball from our repository and copy it over
# ssh to the remote host, where we expand it back
nsd = APP_source_dir()
# Because this could be happening in parallel in various machines
# we generate a tmpfile locally, but the target file is the same
repo_git = APP_repo_git()
if not repo_git:
local_file = tempfile.mktemp(".tar.gz")
else:
#In this case the compression is done after git archive
local_file = tempfile.mktemp(".tar")
create_sources_tarball(local_file)
if repo_git:
local_file += ".gz"
# transfer the tar file if not local
if not is_localhost():
target_tarfile = '/tmp/{0}_tmp.tar'.format(APP_name())
put(local_file, target_tarfile)
else:
target_tarfile = local_file
# unpack the tar file into the APP_src_dir
# (mind the "p", to preserve permissions)
run('mkdir -p {0}'.format(nsd))
with cd(nsd):
run('tar xpf {0}'.format(target_tarfile))
if not is_localhost():
run('rm {0}'.format(target_tarfile))
# Cleaning up now
local('rm {0}'.format(local_file))
success("{0} sources copied".format(APP_name()))
def virtualenv(command, **kwargs):
"""
Just a helper function to execute commands in the APP virtualenv
"""
nid = APP_install_dir()
return run('source {0}/bin/activate && {1}'.format(nid, command), **kwargs)
def unicorn():
with settings(user=env.APP_USER):
run('pkill gunicorn')
def get_linux_flavor():
"""
Obtain and set the env variable linux_flavor
"""
# Already ran through this method
if 'linux_flavor' in env:
return env.linux_flavor
linux_flavor = None
# Try lsb_release
if check_command('lsb_release'):
distributionId = run('lsb_release -i')
if distributionId and distributionId.find(':') != -1:
linux_flavor = distributionId.split(':')[1].strip()
# Try python
if not linux_flavor and check_command('python'):
lf = run(
"python -c 'import platform; print(platform.linux_distribution()[0])'"
)
if lf:
linux_flavor = lf.split()[0]
# Try /etc/issue
if not linux_flavor and check_path('/etc/issue') == '1':
re = run('cat /etc/issue')
issue = re.split()
if issue:
if issue[0] == 'CentOS' or issue[0] == 'Ubuntu' \
or issue[0] == 'Debian':
linux_flavor = issue[0]
elif issue[0] == 'Amazon':
linux_flavor = ' '.join(issue[:2])
elif issue[2] in ('SUSE', 'openSUSE'):
linux_flavor = issue[2]
# Try uname -s
if not linux_flavor:
linux_flavor = run('uname -s')
# Sanitize
if linux_flavor and type(linux_flavor) == type([]):
linux_flavor = linux_flavor[0]
# Final check
if not linux_flavor or linux_flavor not in SUPPORTED_OS:
puts('>>>>>>>>>>')
puts(
'Target machine is running an unsupported or unkown Linux flavor: {0}.'
.format(linux_flavor))
puts('If you know better, please enter it below.')
puts('Must be one of:')
puts(' '.join(SUPPORTED_OS))
linux_flavor = prompt('LINUX flavor: ')
if linux_flavor == 'Linux' and env.docker:
linux_flavor = 'CentOS'
env.linux_flavor = linux_flavor
puts(blue("Remote machine running %s" % linux_flavor))
return linux_flavor
def cleanup():
run('rm -rf daliuge_*')
run('rm -rf DALIUGE')
run('if [ -f .bash_profile.orig ]; then mv .bash_profile.orig .bash_profile; fi'
)
def start_jupyter():
"""
Starts the jupyter server in the correct directory
"""
with cd('{0}'.format(APP_SRC_DIR_NAME)):
run('jupyter notebook --ip=0.0.0.0')
def install_port(package):
"""
Install a package using macports (Mac OSX)
"""
with settings(warn_only=True):
run('sudo port install {0}'.format(package))
def install_brew(package):
"""
Install a package using homebrew (Mac OSX)
"""
with settings(warn_only=True):
run('export HOMEBREW_NO_EMOJI=1; brew install {0} | grep -v "\%"'.format(package))
def check_path(path):
"""
Check existence of remote path
"""
res = run('if [ -e {0} ]; then echo 1; else echo 0; fi'.format(path))
return res
