def install_system_packages(): """ Perform the installation of system-level packages needed by APP to work. """ # Install required packages linux_flavor = get_linux_flavor() if (linux_flavor in ['CentOS', 'Amazon Linux']): # Update the machine completely errmsg = sudo('yum --assumeyes --quiet update', combine_stderr=True, warn_only=True) processCentOSErrMsg(errmsg) install_yum(env.pkgs['YUM_PACKAGES']) if linux_flavor == 'CentOS': sudo('/etc/init.d/iptables stop') # CentOS firewall blocks APP port! elif (linux_flavor in ['Ubuntu', 'Debian']): errmsg = sudo('apt-get -qq -y update', combine_stderr=True, warn_only=True) install_apt(env.pkgs['APT_PACKAGES']) elif linux_flavor in ['SUSE', 'SLES-SP2', 'SLES-SP3', 'SLES', 'openSUSE']: errmsg = sudo('zypper -n -q patch', combine_stderr=True, warn_only=True) install_zypper(env.pkgs['SLES_PACKAGES']) elif linux_flavor == 'Darwin': pkg_mgr = check_brew_port() if pkg_mgr is None: install_homebrew() pkg_mgr = 'brew' if pkg_mgr == 'brew': for package in env.pkgs['BREW_PACKAGES']: install_brew(package) elif pkg_mgr == 'port': for package in env.pkgs['PORT_PACKAGES']: install_port(package) else: abort("Unsupported linux flavor detected: {0}".format(linux_flavor))
def npm_install(): """ Install Node.js to get access to the npm package manager required by pyastrovis NOTE: The code below works only for Debian style Linux """ sudo('curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -') sudo('sudo apt-get install -y nodejs')
def install_jupyterHub(): """ Install the tiniest JupyterHub """ cmd = """curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \ | python3 - --admin jupyter""" sudo(cmd)
def sysinitstart_RASVAMT_and_check_status(): """ Starts the APP daemon process and checks that the server is up and running then it shuts down the server """ # We sleep 2 here as it was found on Mac deployment to docker container # that the shell would exit before the APPDaemon could detach, thus # resulting in no startup self. # # Please replace following line with something meaningful # virtualenv('ngamsDaemon start -cfg {0} && sleep 2'.format(tgt_cfg)) env.APP_INSTALL_DIR = os.path.abspath(os.path.join(home(), APP_INSTALL_DIR_NAME)) env.APP_ROOT_DIR = os.path.abspath(os.path.join(home(), APP_ROOT_DIR_NAME)) env.APP_SRC_DIR = os.path.abspath(os.path.join(home(), APP_SRC_DIR_NAME)) info('Start {0} and check'.format(APP)) start_unicorn() with settings(user=env.AWS_SUDO_USER): sudo('service nginx start') try: u = urllib2.urlopen('http://{0}/static/html/index.html'. format(env.host_string)) except urllib2.URLError: red("RASVAMT NOT running!") return r = u.read() u.close() assert r.find('rasvamt-s-user-documentation') > -1, red("RASVAMT NOT running")
def sysinitstart_APP_and_check_status(): """ Starts the APP daemon process and checks that the server is up and running then it shuts down the server """ sudo('service dlg-nm start') sudo('service dlg-dim start')
def install_sysv_init_script(nsd, nuser, cfgfile): """ Install the init script for an operational deployment of RASVAMT. The init script is an old System V init system. In the presence of a systemd-enabled system we use the update-rc.d tool to enable the script as part of systemd (instead of the System V chkconfig tool which we use instead). The script is prepared to deal with both tools. """ with settings(user=env.AWS_SUDO_USER): print(red("Initialising deployment")) sudo('usermod -a -G {} ec2-user'.format(env.APP_USER)) sudo('mkdir -p /etc/supervisor/') sudo('mkdir -p /etc/supervisor/conf.d/') sudo('cp {0}/fabfile/init/sysv/nginx.conf /etc/nginx/.'. format(APP_source_dir())) # copy nginx and supervisor conf files sudo('cp {0}/fabfile/init/sysv/rasvama.conf /etc/supervisor/conf.d/.'. format(APP_source_dir())) # create the DB with settings(user=env.APP_USER): virtualenv('cd {0}/db; python create_db.py'.format(env.APP_SRC_DIR)) #check if nginx is running else print(red("Server setup and ready to deploy")) #Think we have success("Init scripts installed")
def sysinitstop_EAGLE(): """ Stops the nginx and uwsgi services if running. NOTE: Requires sudo privilidges """ info('Check and stop {0}'.format(APP)) if env.FAB_TASK == 'docker_image': env.sudo_user = '******' elif env.FAB_TASK == 'aws_deploy': env.sudo_user = env.AWS_SUDO_USER try: u = urllib2.urlopen('http://{0}/static/html/index.html'. format(env.host_string)) except urllib2.URLError: red("EAGLE NOT running!") return r = u.read() u.close() if type(r) == bytes: # required for Python3 r = r.decode('UTF-8') assert r.find('eagle-s-user-documentation') > -1, red("EAGLE NOT running") with settings(user=env.sudo_user): sudo('service nginx stop') sudo('service uwsgi stop')
def sysinitstart_EAGLE_and_check_status(): """ Starts the APP daemon process and checks that the server is up and running then it shuts down the server """ # We sleep 2 here as it was found on Mac deployment to docker container # that the shell would exit before the APPDaemon could detach, thus # resulting in no startup self. # # Please replace following line with something meaningful # virtualenv('ngamsDaemon start -cfg {0} && sleep 2'.format(tgt_cfg)) info('Start {0} and check'.format(APP)) if env.FAB_TASK == 'docker_image': env.sudo_user = '******' elif env.FAB_TASK == 'aws_deploy': env.sudo_user = env.AWS_SUDO_USER with settings(user=env.sudo_user): sudo('service nginx start') sudo('service uwsgi start') try: u = urllib2.urlopen('http://{0}/static/html/index.html'. format(env.host_string)) except urllib2.URLError: red("EAGLE NOT running!") return r = u.read() u.close() if type(r) == bytes: # required for Python3 r = r.decode('UTF-8') if r.find('eagle-s-user-documentation') > -1: red("EAGLE NOT running") else: green("EAGLE running. Connect through: http://{0}/new".format(env.host_string))
def install_docker_compose(): print(">>>>> Installing docker-compose <<<<<<<<") sudo('curl -L "https://github.com/docker/compose/releases/download/1.23.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose') sudo('chmod +x /usr/local/bin/docker-compose') sudo('usermod -aG docker pyvospace') sudo('service docker start') time.sleep(5) sudo("sudo sed -ie 's/# user_allow_other/user_allow_other/g' /etc/fuse.conf")
def install_apt(packages): """ Install packages using APT """ # We need to iterate over each one because if at least one of them # is actually not a package (misspelled, doesn't exist anymore, debian- # or ubuntu-specific, etc) the whole install process would fail # On the other hand there appears to be no flag to ignore these errors # on apt-get (tested on Ubuntu 12.04) for pkg in packages + extra_packages(): sudo('apt-get -qq -y install {0}'.format(pkg))
def ffmpeg_install(): """ Compiles and installs the ffmpeg library required by pyastrovis Only required on AWS Linux instances. Debian has packages. """ with cd('/tmp'): sudo('wget https://ffmpeg.org/releases/ffmpeg-snapshot.tar.bz2') sudo('tar -vxjf ./ffmpeg-snapshot.tar.bz2') with cd('/tmp/ffmpeg-snapshot'): sudo('./configure') sudo('make') sudo('make install')
def download(url, target=None, root=False): if target is None: parts = urlparse.urlparse(url) target = parts.path.split('/')[-1] if check_command('wget'): cmd = 'wget --no-check-certificate -q -O {0} {1}'.format(target, url) elif check_command('curl'): cmd = 'curl -o {0} {1}'.format(target, url) else: raise Exception("Neither wget nor curl are installed") if root: sudo(cmd) else: run(cmd) return target
def extra_sudo(): """ sudo npm install -g typescript """ if env.FAB_TASK == 'docker_image': env.sudo_user = '******' elif env.FAB_TASK == 'aws_deploy': env.sudo_user = env.AWS_SUDO_USER else: env.sudo_user = env.user # fall back to current user with settings(user=env.sudo_user): run('curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash') # sudo('curl -sL https://rpm.nodesource.com/setup_10.x |sudo bash -') sudo('yum --assumeyes --quiet install nodejs') sudo('npm install -g typescript')
def install_yum(packages): """ Install packages using YUM """ errmsg = sudo('yum --assumeyes --quiet install {0}'.format(' '.join(packages + extra_packages())),\ combine_stderr=True, warn_only=True) processCentOSErrMsg(errmsg)
def sysinitstart_NGAS_and_check_status(): """ Starts the ngamsDaemon process and checks that the server is up and running. Then it shuts down the server """ # We sleep 2 here as it was found on Mac deployment to docker container that the # shell would exit before the ngasDaemon could detach, thus resulting in no startup. sudo('service ngas-server start && sleep 2') try: res = sudo('service ngas-server status', warn_only=True) print(res) if res.failed: failure( "Couldn't contact NGAS server after starting it. " "Check log files under %s/log/ to find out what went wrong" % APP_source_dir(), with_stars=False) else: success('NGAS server started correctly :)') finally: info("Shutting NGAS server down now") sudo("service ngas-server stop ")
def check_apt(package): """ Check whether package is installed using APT NOTE: This requires sudo access """ # TODO with hide('stdout', 'running'): res = sudo('dpkg -L | grep {0}'.format(package)) if res.find(package) > -1: print( "Installed package {0}".format(package)) return True else: print( "NOT installed package {0}".format(package)) return False
def check_yum(package): """ Check whether package is installed or not NOTE: requires sudo access to machine """ with hide('stdout', 'running', 'stderr'): res = sudo('yum --assumeyes --quiet list installed {0}'.format(package),\ combine_stderr=True, warn_only=True) # print res if res.find(package) > 0: print("Installed package {0}".format(package)) return True else: print("NOT installed package {0}".format(package)) return False
def assign_ddns(): """ Installs the noip ddns client to the specified host. After the installation the configuration step is executed and that requires some manual input. Then the noip2 client is started in background. """ with cd('/usr/local/src'): download('http://www.no-ip.com/client/linux/noip-duc-linux.tar.gz', root=True) sudo('tar xf noip-duc-linux.tar.gz') sudo('cd noip-2.1.9-1') sudo('make install') sudo('noip2 -C') # TODO: put correct startup script in repo and install it # sudo('cp redhat.noip.sh /etc/init.d/noip') # sudo('chmod a+x /etc/init.d/noip') # sudo('chkconfig noip on') # sudo('service noip start') puts(green("\n***** Dynamic IP address assigned ******\n"))
def install_sysv_init_script(nsd, nuser, cfgfile): """ Install the NGAS init script for an operational deployment. The init script is an old System V init system. In the presence of a systemd-enabled system we use the update-rc.d tool to enable the script as part of systemd (instead of the System V chkconfig tool which we use instead). The script is prepared to deal with both tools. """ # Different distros place it in different directories # The init script is prepared for both opt_file = '/etc/sysconfig/ngas' if get_linux_flavor() in ('Ubuntu', 'Debian'): opt_file = '/etc/default/ngas' # Script file installation sudo('cp %s/fabfile/init/sysv/ngas-server /etc/init.d/' % (nsd, )) sudo('chmod 755 /etc/init.d/ngas-server') # Options file installation and edition ntype = settings['NGAS_SERVER_TYPE'] sudo('cp %s/fabfile/init/sysv/ngas-server.options %s' % (nsd, opt_file)) sudo('chmod 644 %s' % (opt_file, )) sed(opt_file, '^USER=.*', 'USER=%s' % (nuser, ), use_sudo=True, backup='') sed(opt_file, '^CFGFILE=.*', 'CFGFILE=%s' % (cfgfile, ), use_sudo=True, backup='') if ntype == 'cache': sed(opt_file, '^CACHE=.*', 'CACHE=YES', use_sudo=True, backup='') elif ntype == 'data-mover': sed(opt_file, '^DATA_MOVER=.*', 'DATA_MOVER=YES', use_sudo=True, backup='') # Enabling init file on boot if check_command('update-rc.d'): sudo('update-rc.d ngas-server defaults') else: sudo('chkconfig --add ngas-server') success("NGAS init script installed")
def postfix_config(): """ Setup a valid Postfix configuration to be used by APP. """ if 'gmail_account' not in env: prompt('GMail Account:', 'gmail_account') if 'gmail_password' not in env: prompt('GMail Password:'******'gmail_password') # Make sure postfix is there and not sendmail sudo('service sendmail stop') sudo('service postfix stop') sudo('chkconfig sendmail off') sudo('chkconfig sendmail --del') sudo('chkconfig postfix --add') sudo('chkconfig postfix on') # Set up the main configuration file and the password file puts(pkg_resources.resource_filename(__name__, 'main.cf'), '/etc/postfix/main.cf') # @UndefinedVariable sudo( 'echo "[smtp.gmail.com]:587 {0}@gmail.com:{1}" > /etc/postfix/sasl_passwd' .format(env.gmail_account, env.gmail_password)) sudo('chmod 400 /etc/postfix/sasl_passwd') sudo('postmap /etc/postfix/sasl_passwd') # Start it sudo('service postfix start')
def install_sysv_init_script(nsd, nuser, cfgfile): """ Install the APP init script for an operational deployment. The init script is an old System V init system. In the presence of a systemd-enabled system we use the update-rc.d tool to enable the script as part of systemd (instead of the System V chkconfig tool which we use instead). The script is prepared to deal with both tools. """ # Different distros place it in different directories # The init script is prepared for both opt_file = '/etc/sysconfig/dlg' if get_linux_flavor() in ('Ubuntu', 'Debian'): opt_file = '/etc/default/dlg' # Script file installation sudo('cp {0}/fabfile/init/sysv/dlg-* /etc/init.d/'.format(nsd)) sudo('chmod 755 /etc/init.d/dlg-*') # Options file installation and edition sudo('cp {0}/fabfile/init/sysv/dlg.options {1}'.format(nsd, opt_file)) sudo('chmod 644 %s' % (opt_file, )) # Enabling init file on boot if check_command('update-rc.d'): sudo('update-rc.d dlg-nm defaults') sudo('update-rc.d dlg-dim defaults') else: sudo('chkconfig --add dlg-nm') sudo('chkconfig --add dlg-dim') success("{0} init script installed".format(env.APP_NAME))
def create_user(user): """ Creates a user in the system. """ # TODO: Check if the user exists # Also, these commands are linux-specific, # there are others that work on MacOS group = user.lower() sudo('groupadd ' + group, warn_only=True) sudo('useradd -g {0} -m -s /bin/bash {1}'.format(group, user), warn_only=True) sudo('mkdir /home/{0}/.ssh'.format(user), warn_only=True) sudo('chmod 700 /home/{0}/.ssh'.format(user)) sudo('chown -R {0}:{1} /home/{0}/.ssh'.format(user, group)) # Copy the public key of our SSH key if we're using one public_key = get_fab_public_key() if public_key: sudo("echo '{0}' >> /home/{1}/.ssh/authorized_keys".format( public_key, user)) sudo('chmod 600 /home/{0}/.ssh/authorized_keys'.format(user)) sudo('chown {0}:{1} /home/{0}/.ssh/authorized_keys'.format( user, group)) # openSUSE creates a suboptimal ~/.profile because it shows an error message # if /etc/profile doesn't exist (which is the case on openSUES dockers), # so we comment out that particular line if get_linux_flavor() == 'openSUSE': with settings(user=user): run('''sed -i 's/^test -z "$PROFILEREAD".*/#\\0/' ~/.profile ''')
def install_sysv_init_script(nsd, nuser, cfgfile): """ Install the uwsgi init script for an operational deployment of EAGLE. The init script is an old System V init system. In the presence of a systemd-enabled system we use the update-rc.d tool to enable the script as part of systemd (instead of the System V chkconfig tool which we use instead). The script is prepared to deal with both tools. """ if env.FAB_TASK == 'docker_image': env.sudo_user = '******' elif env.FAB_TASK == 'aws_deploy': env.sudo_user = env.AWS_SUDO_USER with settings(user=env.sudo_user): # Different distros place it in different directories # The init script is prepared for both opt_file = '/etc/uwsgi/uwsgi.ini' # The uwsgi binary got installed into the virtualenv. Lets pull that over # to the system wide folder. sudo('cp {0}/bin/uwsgi /usr/local/bin/uwsgi'.format(APP_install_dir())) sudo('chmod 755 /usr/local/bin/uwsgi') # init file installation sudo('cp {0}/fabfile/init/sysv/uwsgi /etc/init.d/'.format(APP_source_dir())) sudo('chmod 755 /etc/init.d/uwsgi') # Options file installation and edition sudo('mkdir -p /etc/uwsgi') sudo('cp {0}/fabfile/init/sysv/uwsgi.ini {1}'.format(APP_source_dir(), opt_file)) sudo('chmod 644 {0}'.format(opt_file)) # Enabling init file on boot if check_command('update-rc.d'): sudo('update-rc.d uwsgi defaults') else: sudo('chkconfig --add uwsgi') # Nginx is not in standard repos # go get it [This is just for centos] sudo('cp {0}/fabfile/init/nginx.repo /etc/yum.repos.d/.'. format(APP_source_dir())) sudo('yum update ; yum install nginx') # Now let's connect that to nginx # Copy main nginx conf file sudo('cp {0}/fabfile/init/sysv/nginx.conf /etc/nginx/.'. format(APP_source_dir())) # copy uwsgi nginx conf file sudo('cp {0}/fabfile/init/sysv/eagle.conf /etc/nginx/conf.d/.'. format(APP_source_dir())) success("Init scripts installed")
def install_zypper(packages): """ Install packages using zypper (SLES) """ sudo('zypper --non-interactive install {0}'.format(' '.join(packages + extra_packages())),\ combine_stderr=True, warn_only=True)