def check_brew_cellar(): """ Find the brewing cellar (Mac OSX) """ with hide('output'): # This yields something like "Homebrew 1.4.2-14-g3e99504" # but might be different in earlier versions (or newer) # so we do what we can to clean it up version = run('brew --version') if 'Homebrew ' in version: version = version.split()[1].strip() if '-' in version: version = version.split('-')[0].strip() version = tuple(map(int, version.split('.'))) # I'm not sure exactly when --cellar was introduced, but it was already # there in 0.9.9. On the other hand HOMEBREW_CELLAR was also still # outputted via brew config in 0.9.9, so I think it's safe to make the # cut at 1.0. if version >= (1, 0): cellar = run('brew --cellar') else: cellar = run('brew config | grep HOMEBREW_CELLAR').split(":")[1].strip() return cellar
def create_user(user): """ Creates a user in the system. """ # TODO: Check if the user exists # Also, these commands are linux-specific, # there are others that work on MacOS group = user.lower() sudo('groupadd ' + group, warn_only=True) sudo('useradd -g {0} -m -s /bin/bash {1}'.format(group, user), warn_only=True) sudo('mkdir /home/{0}/.ssh'.format(user), warn_only=True) sudo('chmod 700 /home/{0}/.ssh'.format(user)) sudo('chown -R {0}:{1} /home/{0}/.ssh'.format(user, group)) # Copy the public key of our SSH key if we're using one public_key = get_fab_public_key() if public_key: sudo("echo '{0}' >> /home/{1}/.ssh/authorized_keys".format( public_key, user)) sudo('chmod 600 /home/{0}/.ssh/authorized_keys'.format(user)) sudo('chown {0}:{1} /home/{0}/.ssh/authorized_keys'.format( user, group)) # openSUSE creates a suboptimal ~/.profile because it shows an error message # if /etc/profile doesn't exist (which is the case on openSUES dockers), # so we comment out that particular line if get_linux_flavor() == 'openSUSE': with settings(user=user): run('''sed -i 's/^test -z "$PROFILEREAD".*/#\\0/' ~/.profile ''')
def start_unicorn(): """ Starts the gunicorn daemon which in turn will be called by nginx. """ HOME = home() env.APP_INSTALL_DIR = os.path.abspath(os.path.join(HOME, APP_INSTALL_DIR_NAME)) env.APP_ROOT_DIR = os.path.abspath(os.path.join(HOME, APP_ROOT_DIR_NAME)) env.APP_SRC_DIR = os.path.abspath(os.path.join(HOME, APP_SRC_DIR_NAME)) NAME = "RASVAMT" FLASKDIR = env.APP_INSTALL_DIR+'/src' SOCKFILE = env.APP_INSTALL_DIR+'/sock' PIDFILE = env.APP_INSTALL_DIR+'/gunicorn.pid' USER = env.APP_USER GROUP = env.APP_USER NUM_WORKERS = '3' with settings(user=env.APP_USER): with cd(FLASKDIR): run('{0}/bin/gunicorn main:app -b 127.0.0.1:5000 -D '.format(env.APP_INSTALL_DIR) +\ ' --name {0}'.format(NAME) + \ ' --user={0} --group={1}'.format(USER, GROUP) +\ ' --bind=unix:{0}'.format(SOCKFILE) +\ ' --workers={0}'.format(NUM_WORKERS) +\ ' --log-level=debug --pythonpath={0}'.format(FLASKDIR) +\ ' -p {0}'.format(PIDFILE) )
def prepare_APP_data_dir(): """Creates a new APP root directory""" """ Install the content of the NGAS portal """ if not exists('{0}/ngas/import/'.format(APP_install_dir())): run('mkdir {0}/ngas/import/'.format(APP_install_dir()), warn_only=True) put('data/*.zexp', '{0}/ngas/import/'.format(APP_install_dir())) if not exists('{0}/../NGAS'.format(APP_install_dir())): run('mkdir NGAS', warn_only=True) if not exists('{0}/../NGAS/ngas.sqlite'.format(APP_install_dir())): put('data/ngas.sqlite', '{0}/../NGAS/'.format(APP_install_dir()))
def download(url, target=None, root=False): if target is None: parts = urlparse.urlparse(url) target = parts.path.split('/')[-1] if check_command('wget'): cmd = 'wget --no-check-certificate -q -O {0} {1}'.format(target, url) elif check_command('curl'): cmd = 'curl -o {0} {1}'.format(target, url) else: raise Exception("Neither wget nor curl are installed") if root: sudo(cmd) else: run(cmd) return target
def APP_build_cmd(): env.APP_INSTALL_DIR = os.path.abspath(os.path.join(home(), APP_INSTALL_DIR_NAME)) env.APP_ROOT_DIR = os.path.abspath(os.path.join(home(), APP_ROOT_DIR_NAME)) env.APP_SRC_DIR = os.path.abspath(os.path.join(home(), APP_SRC_DIR_NAME)) build_cmd = [] # create RASVAMT directories and chown to correct user and group run('mkdir -p {0}'.format(env.APP_INSTALL_DIR)) # This not working for some reason # sudo('chown -R {0}:{1} {2}'.format(env.USERS[0], GROUP, APP_INSTALL_DIR)) run('ln -s {0}/src {1}/src'.format(env.APP_SRC_DIR, env.APP_INSTALL_DIR)) return ' '.join(build_cmd)
def check_dir(directory): """ Check existence of remote directory """ res = run( """if [ -d {0} ]; then echo 1; else echo ; fi""".format(directory)) return res
def prepare_ngas_data_dir(): """Creates a new NGAS root directory""" info('Preparing NGAS root directory') nrd = APP_root_dir() tgt_cfg = os.path.join(nrd, 'cfg', 'ngamsServer.conf') with cd(APP_source_dir()): cmd = ['./prepare_ngas_root.sh'] if 'NGAS_OVERWRITE_ROOT' in env and env.NGAS_OVERWRITE_ROOT: cmd.append('-f') cmd.append(nrd) res = run(' '.join(cmd), quiet=True) if res.succeeded: success("NGAS data directory ready") env.tgt_cfg = tgt_cfg return tgt_cfg # Deal with the errors here error = 'NGAS root directory preparation under {0} failed.\n'.format(nrd) if res.return_code == 2: error = (nrd + " already exists. Specify NGAS_OVERWRITE_ROOT to overwrite, " "or a different NGAS_ROOT_DIR location") else: error = res abort(error)
def check_command(command, *args, **kwargs): """ Check existence of command remotely """ res = run( 'if command -v {0} &> /dev/null ;then command -v {0};else echo ;fi'. format(command), *args, **kwargs) return res
def install_user_profile(): """ Put the activation of the virtualenv into the login profile of the user unless the APP_DONT_MODIFY_BASHPROFILE environment variable is defined NOTE: This will be executed for the user running APP. """ if run('echo $APP_DONT_MODIFY_BASHPROFILE') or \ 'APP_NO_BASH_PROFILE' in env: return nid = APP_install_dir() nrd = APP_root_dir() with cd("~"): if not exists(".bash_profile_orig"): run('cp .bash_profile .bash_profile_orig', warn_only=True) else: run('cp .bash_profile_orig .bash_profile') script = ('if [ -f "{0}/bin/activate" ]'.format(nid), 'then', ' source "{0}/bin/activate"'.format(nid), 'fi', 'export APP_PREFIX="{0}"'.format(nrd)) run("echo '{0}' >> .bash_profile".format('\n'.join(script))) success("~/.bash_profile edited for automatic virtualenv sourcing")
def install_homebrew(): """ Task to install homebrew on Mac OSX. NOTE: This should not be done if macports is installed already. """ lf = get_linux_flavor() if lf != 'Darwin': puts(red("Potentially this is not a Mac OSX installation: {0}".format(lf))) raise(ValueError) if check_command('port'): puts(red('MacPorts is installed and it is not recommended to mix it with homebrew!!')) puts(red('Bailing out!')) raise(ValueError) return if not check_command('brew'): run('ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"') else: puts(red('Homebrew is installed already! New installation not required.'))
def check_user(user): """ Task checking existence of user """ res = run( 'if id -u "{0}" >/dev/null 2>&1; then echo 1; else echo 0; fi;'.format( user)) if res == '0': puts('User {0} does not exist'.format(user)) return False else: return True
def cleanup_container(): # Clean downloaded packages, remove unnecessary packages # # This is obviously a CentOS 7 hardcoded list, but we already hardcode # CentOS 7 as the FROM image in our build file so we are basically building # up on that assumption. Generalising all this logic would require quite # some effort. but since it is not necessarily something we need or want, it # is kind of ok to live with this sin. for pkg in ('autoconf', 'bzip2-devel', 'cpp', 'groff-base', 'krb5-devel', 'less', 'libcom_err-devel', 'libgnome-keyring', 'libedit', 'libgomp', 'libkadm5', 'libselinux-devel', 'm4', 'mpfr', 'pcre-devel', 'rsync', 'libverto-devel', 'libmpc', 'gcc', 'gdbm-devel', 'git', 'glibc-devel', 'glibc-headers', 'kernel-headers', 'libdb-devel', 'make', 'openssl-devel', 'patch', 'perl', 'postgresql', 'postgresql-libs', 'python-devel', 'readline-devel', 'sqlite-devel', 'sudo', 'wget', 'zlib-devel', 'libffi-devel'): run('yum --assumeyes --quiet remove %s' % (pkg, ), warn_only=True) run('yum clean all') # Remove user directories that are not needed anymore with settings(user=APP_user()): # By default we do not ship the image with a working APP directory to_remove = ['~/.cache'] if not docker_keep_APP_src(): to_remove.append(APP_source_dir()) if not docker_keep_APP_root(): to_remove.append(APP_root_dir()) for d in to_remove: run('rm -rf %s' % d, )
def virtualenv_setup(): """ Creates a new virtualenv that will hold the APP installation """ APPInstallDir = APP_install_dir() if check_dir(APPInstallDir): overwrite = APP_overwrite_installation() if not overwrite: msg = ("%s exists already. Specify APP_OVERWRITE_INSTALLATION " "to overwrite, or a different APP_INSTALL_DIR location") abort(msg % (APPInstallDir,)) run("rm -rf %s" % (APPInstallDir,)) # Check which python will be bound to the virtualenv ppath = check_python() if not ppath: ppath = python_setup(os.path.join(home(), 'python')) # Use our create_venv.sh script to create the virtualenv # It already handles the download automatically if no virtualenv command is # found in the system, and also allows to specify a python executable path script_path = os.path.dirname(os.path.realpath(__file__))+'/create_venv.sh' put(script_path, APP_source_dir()+'/create_venv.sh') with cd(APP_source_dir()): run("/bin/bash create_venv.sh -p {0} {1}".format(ppath, APPInstallDir)) # Download this particular certifcate; otherwise pip complains # in some platforms if APP_use_custom_pip_cert(): if not(check_dir('~/.pip')): run('mkdir ~/.pip') with cd('~/.pip'): download('http://curl.haxx.se/ca/cacert.pem') run('echo "[global]" > ~/.pip/pip.conf; echo ' '"cert = {0}/.pip/cacert.pem" >> ~/.pip/pip.conf;'.format(home())) # Update pip and install wheel; this way we can install binary wheels from # PyPI if available (like astropy) # TODO: setuptools and python-daemon are here only because # python-daemon 2.1.2 is having a problem to install via setuptools # but not via pip (see https://pagure.io/python-daemon/issue/2 and # https://pagure.io/python-daemon/issue/3). # When this problem is fixed we'll fix our dependency on python-daemo # to avoid this issue entirely virtualenv('pip install -U pip wheel setuptools python-daemon') success("Virtualenv setup completed")
def prepare_APP_data_dir(): """Creates a new APP root directory""" info('Preparing {0} root directory'.format(env.APP_NAME)) nrd = APP_root_dir() # tgt_cfg = os.path.join(nrd, 'cfg', 'ngamsServer.conf') tgt_cfg = None res = run('mkdir -p {0}'.format(nrd)) with cd(APP_source_dir()): for d in env.APP_DATAFILES: res = run('scp -r {0} {1}/.'.format(d, nrd), quiet=True) if res.succeeded: success("{0} data directory ready".format(env.APP_NAME)) return tgt_cfg # Deal with the errors here error = '{0} root directory preparation under {1} failed.\n'.format( env.APP_NAME, nrd) if res.return_code == 2: error = (nrd + " already exists. Specify APP_OVERWRITE_ROOT to " "overwrite, or a different APP_ROOT_DIR location") else: error = res abort(error)
def python_setup(ppath): """ Ensure that there is the right version of python available If not install it from scratch in user directory. """ with cd('/tmp'): download(env.APP_PYTHON_URL) base = os.path.basename(env.APP_PYTHON_URL) pdir = os.path.splitext(base)[0] run('tar -xzf {0}'.format(base)) with cd('/tmp/{0}'.format(pdir)): puts('Python BUILD log-file can be found in: /tmp/py_install.log') puts(green('Configuring Python.....')) run('./configure --prefix {0} > /tmp/py_install.log 2>&1;'.format( ppath)) puts(green('Building Python.....')) run('make >> /tmp/py_install.log 2>&1;') puts(green('Installing Python.....')) run('make install >> /tmp/py_install.log 2>&1') ppath = '{0}/bin/python{1}'.format(ppath, env.APP_PYTHON_VERSION) return ppath
def APP_build_cmd(): # The installation of the bsddb package (needed by ngamsCore) is in # particular difficult because it requires some flags to be passed on # (particularly if using MacOSX's port build_cmd = [] linux_flavor = get_linux_flavor() if linux_flavor == 'Darwin': pkgmgr = check_brew_port() if pkgmgr == 'brew': cellardir = check_brew_cellar() db_version = run( 'ls -tr1 {0}/berkeley-db@4'.format(cellardir)).split()[-1] db_dir = '{0}/berkeley-db@4/{1}'.format(cellardir, db_version) build_cmd.append('BERKELEYDB_DIR={0}'.format(db_dir)) if not settings['NGAS_NO_CLIENT']: build_cmd.append('CFLAGS=-I{0}/include'.format(db_dir)) build_cmd.append('LDFLAGS=-L{0}/lib'.format(db_dir)) else: incdir = MACPORT_DIR + '/include/db60' libdir = MACPORT_DIR + '/lib/db60' build_cmd.append('BERKELEYDB_INCDIR=' + incdir) build_cmd.append('BERKELEYDB_LIBDIR=' + libdir) if not settings['NGAS_NO_CLIENT']: build_cmd.append('CFLAGS=-I' + incdir) build_cmd.append('LDFLAGS=-L' + libdir) build_cmd.append( 'YES_I_HAVE_THE_RIGHT_TO_USE_THIS_BERKELEY_DB_VERSION=1') if settings['NGAS_NO_CRC32C']: build_cmd.append('NGAS_NO_CRC32C=1') build_cmd.append('./build.sh') if not settings['NGAS_NO_CLIENT']: build_cmd.append("-c") if settings['NGAS_DEVELOP']: build_cmd.append("-d") if not settings['NGAS_DOC_DEPENDENCIES']: build_cmd.append('-D') return ' '.join(build_cmd)
def copy_sources(): """ Creates a copy of the APP sources in the target host. """ # We don't open all the git repositories to the world, so for the time # being we always make a tarball from our repository and copy it over # ssh to the remote host, where we expand it back nsd = APP_source_dir() # Because this could be happening in parallel in various machines # we generate a tmpfile locally, but the target file is the same repo_git = APP_repo_git() if not repo_git: local_file = tempfile.mktemp(".tar.gz") else: #In this case the compression is done after git archive local_file = tempfile.mktemp(".tar") create_sources_tarball(local_file) if repo_git: local_file += ".gz" # transfer the tar file if not local if not is_localhost(): target_tarfile = '/tmp/{0}_tmp.tar'.format(APP_name()) put(local_file, target_tarfile) else: target_tarfile = local_file # unpack the tar file into the APP_src_dir # (mind the "p", to preserve permissions) run('mkdir -p {0}'.format(nsd)) with cd(nsd): run('tar xpf {0}'.format(target_tarfile)) if not is_localhost(): run('rm {0}'.format(target_tarfile)) # Cleaning up now local('rm {0}'.format(local_file)) success("{0} sources copied".format(APP_name()))
def virtualenv(command, **kwargs): """ Just a helper function to execute commands in the APP virtualenv """ nid = APP_install_dir() return run('source {0}/bin/activate && {1}'.format(nid, command), **kwargs)
def unicorn(): with settings(user=env.APP_USER): run('pkill gunicorn')
def get_linux_flavor(): """ Obtain and set the env variable linux_flavor """ # Already ran through this method if 'linux_flavor' in env: return env.linux_flavor linux_flavor = None # Try lsb_release if check_command('lsb_release'): distributionId = run('lsb_release -i') if distributionId and distributionId.find(':') != -1: linux_flavor = distributionId.split(':')[1].strip() # Try python if not linux_flavor and check_command('python'): lf = run( "python -c 'import platform; print(platform.linux_distribution()[0])'" ) if lf: linux_flavor = lf.split()[0] # Try /etc/issue if not linux_flavor and check_path('/etc/issue') == '1': re = run('cat /etc/issue') issue = re.split() if issue: if issue[0] == 'CentOS' or issue[0] == 'Ubuntu' \ or issue[0] == 'Debian': linux_flavor = issue[0] elif issue[0] == 'Amazon': linux_flavor = ' '.join(issue[:2]) elif issue[2] in ('SUSE', 'openSUSE'): linux_flavor = issue[2] # Try uname -s if not linux_flavor: linux_flavor = run('uname -s') # Sanitize if linux_flavor and type(linux_flavor) == type([]): linux_flavor = linux_flavor[0] # Final check if not linux_flavor or linux_flavor not in SUPPORTED_OS: puts('>>>>>>>>>>') puts( 'Target machine is running an unsupported or unkown Linux flavor: {0}.' .format(linux_flavor)) puts('If you know better, please enter it below.') puts('Must be one of:') puts(' '.join(SUPPORTED_OS)) linux_flavor = prompt('LINUX flavor: ') if linux_flavor == 'Linux' and env.docker: linux_flavor = 'CentOS' env.linux_flavor = linux_flavor puts(blue("Remote machine running %s" % linux_flavor)) return linux_flavor
def cleanup(): run('rm -rf daliuge_*') run('rm -rf DALIUGE') run('if [ -f .bash_profile.orig ]; then mv .bash_profile.orig .bash_profile; fi' )
def start_jupyter(): """ Starts the jupyter server in the correct directory """ with cd('{0}'.format(APP_SRC_DIR_NAME)): run('jupyter notebook --ip=0.0.0.0')
def install_port(package): """ Install a package using macports (Mac OSX) """ with settings(warn_only=True): run('sudo port install {0}'.format(package))
def install_brew(package): """ Install a package using homebrew (Mac OSX) """ with settings(warn_only=True): run('export HOMEBREW_NO_EMOJI=1; brew install {0} | grep -v "\%"'.format(package))
def check_path(path): """ Check existence of remote path """ res = run('if [ -e {0} ]; then echo 1; else echo 0; fi'.format(path)) return res