def install_system_packages():
"""
Perform the installation of system-level packages needed by APP to work.
"""
# Install required packages
linux_flavor = get_linux_flavor()
if (linux_flavor in ['CentOS', 'Amazon Linux']):
# Update the machine completely
errmsg = sudo('yum --assumeyes --quiet update', combine_stderr=True, warn_only=True)
processCentOSErrMsg(errmsg)
install_yum(env.pkgs['YUM_PACKAGES'])
if linux_flavor == 'CentOS':
sudo('/etc/init.d/iptables stop') # CentOS firewall blocks APP port!
elif (linux_flavor in ['Ubuntu', 'Debian']):
errmsg = sudo('apt-get -qq -y update', combine_stderr=True, warn_only=True)
install_apt(env.pkgs['APT_PACKAGES'])
elif linux_flavor in ['SUSE', 'SLES-SP2', 'SLES-SP3', 'SLES', 'openSUSE']:
errmsg = sudo('zypper -n -q patch', combine_stderr=True, warn_only=True)
install_zypper(env.pkgs['SLES_PACKAGES'])
elif linux_flavor == 'Darwin':
pkg_mgr = check_brew_port()
if pkg_mgr is None:
install_homebrew()
pkg_mgr = 'brew'
if pkg_mgr == 'brew':
for package in env.pkgs['BREW_PACKAGES']:
install_brew(package)
elif pkg_mgr == 'port':
for package in env.pkgs['PORT_PACKAGES']:
install_port(package)
else:
abort("Unsupported linux flavor detected: {0}".format(linux_flavor))
def npm_install():
"""
Install Node.js to get access to the npm package manager required by pyastrovis
NOTE: The code below works only for Debian style Linux
"""
sudo('curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -')
sudo('sudo apt-get install -y nodejs')
def install_jupyterHub():
"""
Install the tiniest JupyterHub
"""
cmd = """curl https://raw.githubusercontent.com/jupyterhub/the-littlest-jupyterhub/master/bootstrap/bootstrap.py \
| python3 - --admin jupyter"""
sudo(cmd)
def sysinitstart_RASVAMT_and_check_status():
"""
Starts the APP daemon process and checks that the server is up and running
then it shuts down the server
"""
# We sleep 2 here as it was found on Mac deployment to docker container
# that the shell would exit before the APPDaemon could detach, thus
# resulting in no startup self.
#
# Please replace following line with something meaningful
# virtualenv('ngamsDaemon start -cfg {0} && sleep 2'.format(tgt_cfg))
env.APP_INSTALL_DIR = os.path.abspath(os.path.join(home(), APP_INSTALL_DIR_NAME))
env.APP_ROOT_DIR = os.path.abspath(os.path.join(home(), APP_ROOT_DIR_NAME))
env.APP_SRC_DIR = os.path.abspath(os.path.join(home(), APP_SRC_DIR_NAME))
info('Start {0} and check'.format(APP))
start_unicorn()
with settings(user=env.AWS_SUDO_USER):
sudo('service nginx start')
try:
u = urllib2.urlopen('http://{0}/static/html/index.html'.
format(env.host_string))
except urllib2.URLError:
red("RASVAMT NOT running!")
return
r = u.read()
u.close()
assert r.find('rasvamt-s-user-documentation') > -1, red("RASVAMT NOT running")
def sysinitstart_APP_and_check_status():
"""
Starts the APP daemon process and checks that the server is up and running
then it shuts down the server
"""
sudo('service dlg-nm start')
sudo('service dlg-dim start')
def install_sysv_init_script(nsd, nuser, cfgfile):
"""
Install the init script for an operational deployment of RASVAMT.
The init script is an old System V init system.
In the presence of a systemd-enabled system we use the update-rc.d tool
to enable the script as part of systemd (instead of the System V chkconfig
tool which we use instead). The script is prepared to deal with both tools.
"""
with settings(user=env.AWS_SUDO_USER):
print(red("Initialising deployment"))
sudo('usermod -a -G {} ec2-user'.format(env.APP_USER))
sudo('mkdir -p /etc/supervisor/')
sudo('mkdir -p /etc/supervisor/conf.d/')
sudo('cp {0}/fabfile/init/sysv/nginx.conf /etc/nginx/.'.
format(APP_source_dir()))
# copy nginx and supervisor conf files
sudo('cp {0}/fabfile/init/sysv/rasvama.conf /etc/supervisor/conf.d/.'.
format(APP_source_dir()))
# create the DB
with settings(user=env.APP_USER):
virtualenv('cd {0}/db; python create_db.py'.format(env.APP_SRC_DIR))
#check if nginx is running else
print(red("Server setup and ready to deploy"))
#Think we have
success("Init scripts installed")
def sysinitstop_EAGLE():
"""
Stops the nginx and uwsgi services if running.
NOTE: Requires sudo privilidges
"""
info('Check and stop {0}'.format(APP))
if env.FAB_TASK == 'docker_image':
env.sudo_user = '******'
elif env.FAB_TASK == 'aws_deploy':
env.sudo_user = env.AWS_SUDO_USER
try:
u = urllib2.urlopen('http://{0}/static/html/index.html'.
format(env.host_string))
except urllib2.URLError:
red("EAGLE NOT running!")
return
r = u.read()
u.close()
if type(r) == bytes: # required for Python3
r = r.decode('UTF-8')
assert r.find('eagle-s-user-documentation') > -1, red("EAGLE NOT running")
with settings(user=env.sudo_user):
sudo('service nginx stop')
sudo('service uwsgi stop')
def sysinitstart_EAGLE_and_check_status():
"""
Starts the APP daemon process and checks that the server is up and running
then it shuts down the server
"""
# We sleep 2 here as it was found on Mac deployment to docker container
# that the shell would exit before the APPDaemon could detach, thus
# resulting in no startup self.
#
# Please replace following line with something meaningful
# virtualenv('ngamsDaemon start -cfg {0} && sleep 2'.format(tgt_cfg))
info('Start {0} and check'.format(APP))
if env.FAB_TASK == 'docker_image':
env.sudo_user = '******'
elif env.FAB_TASK == 'aws_deploy':
env.sudo_user = env.AWS_SUDO_USER
with settings(user=env.sudo_user):
sudo('service nginx start')
sudo('service uwsgi start')
try:
u = urllib2.urlopen('http://{0}/static/html/index.html'.
format(env.host_string))
except urllib2.URLError:
red("EAGLE NOT running!")
return
r = u.read()
u.close()
if type(r) == bytes: # required for Python3
r = r.decode('UTF-8')
if r.find('eagle-s-user-documentation') > -1:
red("EAGLE NOT running")
else:
green("EAGLE running. Connect through: http://{0}/new".format(env.host_string))
def install_docker_compose():
print(">>>>> Installing docker-compose <<<<<<<<")
sudo('curl -L "https://github.com/docker/compose/releases/download/1.23.1/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose')
sudo('chmod +x /usr/local/bin/docker-compose')
sudo('usermod -aG docker pyvospace')
sudo('service docker start')
time.sleep(5)
sudo("sudo sed -ie 's/# user_allow_other/user_allow_other/g' /etc/fuse.conf")
def install_apt(packages):
"""
Install packages using APT
"""
# We need to iterate over each one because if at least one of them
# is actually not a package (misspelled, doesn't exist anymore, debian-
# or ubuntu-specific, etc) the whole install process would fail
# On the other hand there appears to be no flag to ignore these errors
# on apt-get (tested on Ubuntu 12.04)
for pkg in packages + extra_packages():
sudo('apt-get -qq -y install {0}'.format(pkg))
def ffmpeg_install():
"""
Compiles and installs the ffmpeg library required by pyastrovis
Only required on AWS Linux instances. Debian has packages.
"""
with cd('/tmp'):
sudo('wget https://ffmpeg.org/releases/ffmpeg-snapshot.tar.bz2')
sudo('tar -vxjf ./ffmpeg-snapshot.tar.bz2')
with cd('/tmp/ffmpeg-snapshot'):
sudo('./configure')
sudo('make')
sudo('make install')
def download(url, target=None, root=False):
if target is None:
parts = urlparse.urlparse(url)
target = parts.path.split('/')[-1]
if check_command('wget'):
cmd = 'wget --no-check-certificate -q -O {0} {1}'.format(target, url)
elif check_command('curl'):
cmd = 'curl -o {0} {1}'.format(target, url)
else:
raise Exception("Neither wget nor curl are installed")
if root:
sudo(cmd)
else:
run(cmd)
return target
def extra_sudo():
"""
sudo npm install -g typescript
"""
if env.FAB_TASK == 'docker_image':
env.sudo_user = '******'
elif env.FAB_TASK == 'aws_deploy':
env.sudo_user = env.AWS_SUDO_USER
else:
env.sudo_user = env.user # fall back to current user
with settings(user=env.sudo_user):
run('curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash')
# sudo('curl -sL https://rpm.nodesource.com/setup_10.x |sudo bash -')
sudo('yum --assumeyes --quiet install nodejs')
sudo('npm install -g typescript')
def install_yum(packages):
"""
Install packages using YUM
"""
errmsg = sudo('yum --assumeyes --quiet install {0}'.format(' '.join(packages + extra_packages())),\
combine_stderr=True, warn_only=True)
processCentOSErrMsg(errmsg)
def sysinitstart_NGAS_and_check_status():
"""
Starts the ngamsDaemon process and checks that the server is up and running.
Then it shuts down the server
"""
# We sleep 2 here as it was found on Mac deployment to docker container that the
# shell would exit before the ngasDaemon could detach, thus resulting in no startup.
sudo('service ngas-server start && sleep 2')
try:
res = sudo('service ngas-server status', warn_only=True)
print(res)
if res.failed:
failure(
"Couldn't contact NGAS server after starting it. "
"Check log files under %s/log/ to find out what went wrong" %
APP_source_dir(),
with_stars=False)
else:
success('NGAS server started correctly :)')
finally:
info("Shutting NGAS server down now")
sudo("service ngas-server stop ")
def check_apt(package):
"""
Check whether package is installed using APT
NOTE: This requires sudo access
"""
# TODO
with hide('stdout', 'running'):
res = sudo('dpkg -L | grep {0}'.format(package))
if res.find(package) > -1:
print( "Installed package {0}".format(package))
return True
else:
print( "NOT installed package {0}".format(package))
return False
def check_yum(package):
"""
Check whether package is installed or not
NOTE: requires sudo access to machine
"""
with hide('stdout', 'running', 'stderr'):
res = sudo('yum --assumeyes --quiet list installed {0}'.format(package),\
combine_stderr=True, warn_only=True)
# print res
if res.find(package) > 0:
print("Installed package {0}".format(package))
return True
else:
print("NOT installed package {0}".format(package))
return False
def assign_ddns():
"""
Installs the noip ddns client to the specified host.
After the installation the configuration step is executed and that
requires some manual input. Then the noip2 client is started in background.
"""
with cd('/usr/local/src'):
download('http://www.no-ip.com/client/linux/noip-duc-linux.tar.gz',
root=True)
sudo('tar xf noip-duc-linux.tar.gz')
sudo('cd noip-2.1.9-1')
sudo('make install')
sudo('noip2 -C')
# TODO: put correct startup script in repo and install it
# sudo('cp redhat.noip.sh /etc/init.d/noip')
# sudo('chmod a+x /etc/init.d/noip')
# sudo('chkconfig noip on')
# sudo('service noip start')
puts(green("\n***** Dynamic IP address assigned ******\n"))
def install_sysv_init_script(nsd, nuser, cfgfile):
"""
Install the NGAS init script for an operational deployment.
The init script is an old System V init system.
In the presence of a systemd-enabled system we use the update-rc.d tool
to enable the script as part of systemd (instead of the System V chkconfig
tool which we use instead). The script is prepared to deal with both tools.
"""
# Different distros place it in different directories
# The init script is prepared for both
opt_file = '/etc/sysconfig/ngas'
if get_linux_flavor() in ('Ubuntu', 'Debian'):
opt_file = '/etc/default/ngas'
# Script file installation
sudo('cp %s/fabfile/init/sysv/ngas-server /etc/init.d/' % (nsd, ))
sudo('chmod 755 /etc/init.d/ngas-server')
# Options file installation and edition
ntype = settings['NGAS_SERVER_TYPE']
sudo('cp %s/fabfile/init/sysv/ngas-server.options %s' % (nsd, opt_file))
sudo('chmod 644 %s' % (opt_file, ))
sed(opt_file, '^USER=.*', 'USER=%s' % (nuser, ), use_sudo=True, backup='')
sed(opt_file,
'^CFGFILE=.*',
'CFGFILE=%s' % (cfgfile, ),
use_sudo=True,
backup='')
if ntype == 'cache':
sed(opt_file, '^CACHE=.*', 'CACHE=YES', use_sudo=True, backup='')
elif ntype == 'data-mover':
sed(opt_file,
'^DATA_MOVER=.*',
'DATA_MOVER=YES',
use_sudo=True,
backup='')
# Enabling init file on boot
if check_command('update-rc.d'):
sudo('update-rc.d ngas-server defaults')
else:
sudo('chkconfig --add ngas-server')
success("NGAS init script installed")
def postfix_config():
"""
Setup a valid Postfix configuration to be used by APP.
"""
if 'gmail_account' not in env:
prompt('GMail Account:', 'gmail_account')
if 'gmail_password' not in env:
prompt('GMail Password:'******'gmail_password')
# Make sure postfix is there and not sendmail
sudo('service sendmail stop')
sudo('service postfix stop')
sudo('chkconfig sendmail off')
sudo('chkconfig sendmail --del')
sudo('chkconfig postfix --add')
sudo('chkconfig postfix on')
# Set up the main configuration file and the password file
puts(pkg_resources.resource_filename(__name__, 'main.cf'),
'/etc/postfix/main.cf') # @UndefinedVariable
sudo(
'echo "[smtp.gmail.com]:587 {0}@gmail.com:{1}" > /etc/postfix/sasl_passwd'
.format(env.gmail_account, env.gmail_password))
sudo('chmod 400 /etc/postfix/sasl_passwd')
sudo('postmap /etc/postfix/sasl_passwd')
# Start it
sudo('service postfix start')
def install_sysv_init_script(nsd, nuser, cfgfile):
"""
Install the APP init script for an operational deployment.
The init script is an old System V init system.
In the presence of a systemd-enabled system we use the update-rc.d tool
to enable the script as part of systemd (instead of the System V chkconfig
tool which we use instead). The script is prepared to deal with both tools.
"""
# Different distros place it in different directories
# The init script is prepared for both
opt_file = '/etc/sysconfig/dlg'
if get_linux_flavor() in ('Ubuntu', 'Debian'):
opt_file = '/etc/default/dlg'
# Script file installation
sudo('cp {0}/fabfile/init/sysv/dlg-* /etc/init.d/'.format(nsd))
sudo('chmod 755 /etc/init.d/dlg-*')
# Options file installation and edition
sudo('cp {0}/fabfile/init/sysv/dlg.options {1}'.format(nsd, opt_file))
sudo('chmod 644 %s' % (opt_file, ))
# Enabling init file on boot
if check_command('update-rc.d'):
sudo('update-rc.d dlg-nm defaults')
sudo('update-rc.d dlg-dim defaults')
else:
sudo('chkconfig --add dlg-nm')
sudo('chkconfig --add dlg-dim')
success("{0} init script installed".format(env.APP_NAME))
def create_user(user):
"""
Creates a user in the system.
"""
# TODO: Check if the user exists
# Also, these commands are linux-specific,
# there are others that work on MacOS
group = user.lower()
sudo('groupadd ' + group, warn_only=True)
sudo('useradd -g {0} -m -s /bin/bash {1}'.format(group, user),
warn_only=True)
sudo('mkdir /home/{0}/.ssh'.format(user), warn_only=True)
sudo('chmod 700 /home/{0}/.ssh'.format(user))
sudo('chown -R {0}:{1} /home/{0}/.ssh'.format(user, group))
# Copy the public key of our SSH key if we're using one
public_key = get_fab_public_key()
if public_key:
sudo("echo '{0}' >> /home/{1}/.ssh/authorized_keys".format(
public_key, user))
sudo('chmod 600 /home/{0}/.ssh/authorized_keys'.format(user))
sudo('chown {0}:{1} /home/{0}/.ssh/authorized_keys'.format(
user, group))
# openSUSE creates a suboptimal ~/.profile because it shows an error message
# if /etc/profile doesn't exist (which is the case on openSUES dockers),
# so we comment out that particular line
if get_linux_flavor() == 'openSUSE':
with settings(user=user):
run('''sed -i 's/^test -z "$PROFILEREAD".*/#\\0/' ~/.profile ''')
def install_sysv_init_script(nsd, nuser, cfgfile):
"""
Install the uwsgi init script for an operational deployment of EAGLE.
The init script is an old System V init system.
In the presence of a systemd-enabled system we use the update-rc.d tool
to enable the script as part of systemd (instead of the System V chkconfig
tool which we use instead). The script is prepared to deal with both tools.
"""
if env.FAB_TASK == 'docker_image':
env.sudo_user = '******'
elif env.FAB_TASK == 'aws_deploy':
env.sudo_user = env.AWS_SUDO_USER
with settings(user=env.sudo_user):
# Different distros place it in different directories
# The init script is prepared for both
opt_file = '/etc/uwsgi/uwsgi.ini'
# The uwsgi binary got installed into the virtualenv. Lets pull that over
# to the system wide folder.
sudo('cp {0}/bin/uwsgi /usr/local/bin/uwsgi'.format(APP_install_dir()))
sudo('chmod 755 /usr/local/bin/uwsgi')
# init file installation
sudo('cp {0}/fabfile/init/sysv/uwsgi /etc/init.d/'.format(APP_source_dir()))
sudo('chmod 755 /etc/init.d/uwsgi')
# Options file installation and edition
sudo('mkdir -p /etc/uwsgi')
sudo('cp {0}/fabfile/init/sysv/uwsgi.ini {1}'.format(APP_source_dir(),
opt_file))
sudo('chmod 644 {0}'.format(opt_file))
# Enabling init file on boot
if check_command('update-rc.d'):
sudo('update-rc.d uwsgi defaults')
else:
sudo('chkconfig --add uwsgi')
# Nginx is not in standard repos
# go get it [This is just for centos]
sudo('cp {0}/fabfile/init/nginx.repo /etc/yum.repos.d/.'.
format(APP_source_dir()))
sudo('yum update ; yum install nginx')
# Now let's connect that to nginx
# Copy main nginx conf file
sudo('cp {0}/fabfile/init/sysv/nginx.conf /etc/nginx/.'.
format(APP_source_dir()))
# copy uwsgi nginx conf file
sudo('cp {0}/fabfile/init/sysv/eagle.conf /etc/nginx/conf.d/.'.
format(APP_source_dir()))
success("Init scripts installed")
def install_zypper(packages):
"""
Install packages using zypper (SLES)
"""
sudo('zypper --non-interactive install {0}'.format(' '.join(packages + extra_packages())),\
combine_stderr=True, warn_only=True)
