def start_APP_and_check_status():
"""
Starts the APP daemon process and checks that the server is up and running
then it shuts down the server
"""
with cd('{0}/pyvospace/server/deploy'.format(APP_source_dir())):
# >>>> Darwin docker shows a permission issue with keychain access <<<<
if get_linux_flavor() != 'Darwin':
virtualenv('docker-compose build')
else:
info('>>>> Darwin reuqires to build docker container manually')
info('>>>> docker-compose build')
virtualenv(
'docker run -d -p 5435:5432 pyvospace/pyvospace-db -h 0.0.0.0')
time.sleep(10)
with cd('{0}'.format(APP_source_dir())):
virtualenv('python -m unittest discover test')
# run('mkdir -p /tmp/fuse')
# virtualenv('posix_space --cfg test_vo.ini > /tmp/space.log 2>&1')
# time.sleep(2)
# virtualenv('posix_storage --cfg test_vo.ini > /tmp/storage.log 2>&1')
# time.sleep(2)
# virtualenv('python -m pyvospace.client.fuse --host localhost --port 8080 --username test --password test --mountpoint /tmp/fuse/`` > /tmp/fusemnt.log 2>&1')
# time.sleep(2)
# run("cd /tmp/fuse && mkdir -p newdir && cd newdir && echo 'Hello World!' >> data && cat data")
success('{0} is working...'.format(env.APP_NAME))
def install_sysv_init_script(nsd, nuser, cfgfile):
"""
Install the uwsgi init script for an operational deployment of EAGLE.
The init script is an old System V init system.
In the presence of a systemd-enabled system we use the update-rc.d tool
to enable the script as part of systemd (instead of the System V chkconfig
tool which we use instead). The script is prepared to deal with both tools.
"""
if env.FAB_TASK == 'docker_image':
env.sudo_user = '******'
elif env.FAB_TASK == 'aws_deploy':
env.sudo_user = env.AWS_SUDO_USER
with settings(user=env.sudo_user):
# Different distros place it in different directories
# The init script is prepared for both
opt_file = '/etc/uwsgi/uwsgi.ini'
# The uwsgi binary got installed into the virtualenv. Lets pull that over
# to the system wide folder.
sudo('cp {0}/bin/uwsgi /usr/local/bin/uwsgi'.format(APP_install_dir()))
sudo('chmod 755 /usr/local/bin/uwsgi')
# init file installation
sudo('cp {0}/fabfile/init/sysv/uwsgi /etc/init.d/'.format(APP_source_dir()))
sudo('chmod 755 /etc/init.d/uwsgi')
# Options file installation and edition
sudo('mkdir -p /etc/uwsgi')
sudo('cp {0}/fabfile/init/sysv/uwsgi.ini {1}'.format(APP_source_dir(),
opt_file))
sudo('chmod 644 {0}'.format(opt_file))
# Enabling init file on boot
if check_command('update-rc.d'):
sudo('update-rc.d uwsgi defaults')
else:
sudo('chkconfig --add uwsgi')
# Nginx is not in standard repos
# go get it [This is just for centos]
sudo('cp {0}/fabfile/init/nginx.repo /etc/yum.repos.d/.'.
format(APP_source_dir()))
sudo('yum update ; yum install nginx')
# Now let's connect that to nginx
# Copy main nginx conf file
sudo('cp {0}/fabfile/init/sysv/nginx.conf /etc/nginx/.'.
format(APP_source_dir()))
# copy uwsgi nginx conf file
sudo('cp {0}/fabfile/init/sysv/eagle.conf /etc/nginx/conf.d/.'.
format(APP_source_dir()))
success("Init scripts installed")
def prepare_ngas_data_dir():
"""Creates a new NGAS root directory"""
info('Preparing NGAS root directory')
nrd = APP_root_dir()
tgt_cfg = os.path.join(nrd, 'cfg', 'ngamsServer.conf')
with cd(APP_source_dir()):
cmd = ['./prepare_ngas_root.sh']
if 'NGAS_OVERWRITE_ROOT' in env and env.NGAS_OVERWRITE_ROOT:
cmd.append('-f')
cmd.append(nrd)
res = run(' '.join(cmd), quiet=True)
if res.succeeded:
success("NGAS data directory ready")
env.tgt_cfg = tgt_cfg
return tgt_cfg
# Deal with the errors here
error = 'NGAS root directory preparation under {0} failed.\n'.format(nrd)
if res.return_code == 2:
error = (nrd +
" already exists. Specify NGAS_OVERWRITE_ROOT to overwrite, "
"or a different NGAS_ROOT_DIR location")
else:
error = res
abort(error)
def cleanup_container():
# Clean downloaded packages, remove unnecessary packages
#
# This is obviously a CentOS 7 hardcoded list, but we already hardcode
# CentOS 7 as the FROM image in our build file so we are basically building
# up on that assumption. Generalising all this logic would require quite
# some effort. but since it is not necessarily something we need or want, it
# is kind of ok to live with this sin.
for pkg in ('autoconf', 'bzip2-devel', 'cpp', 'groff-base', 'krb5-devel',
'less', 'libcom_err-devel', 'libgnome-keyring', 'libedit',
'libgomp', 'libkadm5', 'libselinux-devel', 'm4', 'mpfr',
'pcre-devel', 'rsync', 'libverto-devel', 'libmpc', 'gcc',
'gdbm-devel', 'git', 'glibc-devel', 'glibc-headers',
'kernel-headers', 'libdb-devel', 'make', 'openssl-devel',
'patch', 'perl', 'postgresql', 'postgresql-libs',
'python-devel', 'readline-devel', 'sqlite-devel', 'sudo',
'wget', 'zlib-devel', 'libffi-devel'):
run('yum --assumeyes --quiet remove %s' % (pkg, ), warn_only=True)
run('yum clean all')
# Remove user directories that are not needed anymore
with settings(user=APP_user()):
# By default we do not ship the image with a working APP directory
to_remove = ['~/.cache']
if not docker_keep_APP_src():
to_remove.append(APP_source_dir())
if not docker_keep_APP_root():
to_remove.append(APP_root_dir())
for d in to_remove:
run('rm -rf %s' % d, )
def build_APP():
"""
Builds and installs APP into the target virtualenv.
"""
with cd(APP_source_dir()):
develop = False
no_doc_dependencies = APP_doc_dependencies()
build_cmd = APP_build_cmd()
print(build_cmd)
if build_cmd != '':
virtualenv(build_cmd)
with cd(APP_install_dir()):
virtualenv(
'pip install --no-binary zc.recipe.egg -r {0}'.format(ZOPE_URL))
# virtualenv('pip install -U zope.interface')
virtualenv('pip install Products.ExternalMethod')
virtualenv('pip install Products.PythonScripts')
virtualenv('pip install Products.ZSQLMethods==2.13.5')
# virtualenv('easy_install Products.SQLAlchemyDA')
virtualenv('pip install psycopg2-binary')
virtualenv('mkzopeinstance -d {0}/ngas -u {1}:{2}'.format(
APP_install_dir(), 'admin', 'admin4zope'))
with cd('/tmp'):
virtualenv(
'git clone https://github.com/zopefoundation/Products.SQLAlchemyDA.git SQLAlchemyDA'
)
virtualenv('cd SQLAlchemyDA; python setup.py install')
def build_APP():
"""
Builds and installs APP into the target virtualenv.
"""
with cd(APP_source_dir()):
extra_pkgs = extra_python_packages()
if extra_pkgs:
virtualenv('pip install %s' % ' '.join(extra_pkgs))
develop = False
no_doc_dependencies = APP_doc_dependencies()
build_cmd = APP_build_cmd()
print(build_cmd)
if build_cmd != '':
virtualenv(build_cmd)
success("{0} built and installed".format(env.APP_NAME))
def APP_build_cmd():
# The installation of the bsddb package (needed by ngamsCore) is in
# particular difficult because it requires some flags to be passed on
# (particularly if using MacOSX's port
# >>>> NOTE: This function potentially needs heavy customisation <<<<<<
build_cmd = []
# linux_flavor = get_linux_flavor()
env.APP_INSTALL_DIR = os.path.abspath(os.path.join(home(), APP_INSTALL_DIR_NAME))
env.APP_ROOT_DIR = os.path.abspath(os.path.join(home(), APP_ROOT_DIR_NAME))
env.APP_SRC_DIR = os.path.abspath(os.path.join(home(), APP_SRC_DIR_NAME))
build_cmd.append('cd {0} ;'.format(APP_source_dir()))
build_cmd.append(
'curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash ;')
build_cmd.append('source ~/.nvm/nvm.sh')
build_cmd.append('nvm install node')
# build_cmd.append('npm install typescript')
build_cmd.append('pip install . ;')
return ' '.join(build_cmd)
def sysinitstart_APP_and_check_status():
"""
Starts the APP daemon process and checks that the server is up and running
then it shuts down the server
"""
###>>>
# The following just runs the DB in a docker container and runs the tests
###<<<
nuser = APP_user()
with settings(user=nuser):
with cd('{0}'.format(APP_source_dir())):
# virtualenv('python -m unittest discover test')
# run('mkdir -p /tmp/fuse')
# virtualenv('posix_space --cfg test_vo.ini > /tmp/space.log 2>&1')
# time.sleep(2)
# virtualenv('posix_storage --cfg test_vo.ini > /tmp/storage.log 2>&1')
# time.sleep(2)
# virtualenv('python -m pyvospace.client.fuse --host localhost --port 8080 --username test --password test --mountpoint /tmp/fuse/`` > /tmp/fusemnt.log 2>&1')
# time.sleep(2)
# run("cd /tmp/fuse && mkdir -p newdir && cd newdir && echo 'Hello World!' >> data && cat data")
pass
success("{0} successfully tested!".format(env.APP_NAME))
def sysinitstart_NGAS_and_check_status():
"""
Starts the ngamsDaemon process and checks that the server is up and running.
Then it shuts down the server
"""
# We sleep 2 here as it was found on Mac deployment to docker container that the
# shell would exit before the ngasDaemon could detach, thus resulting in no startup.
sudo('service ngas-server start && sleep 2')
try:
res = sudo('service ngas-server status', warn_only=True)
print(res)
if res.failed:
failure(
"Couldn't contact NGAS server after starting it. "
"Check log files under %s/log/ to find out what went wrong" %
APP_source_dir(),
with_stars=False)
else:
success('NGAS server started correctly :)')
finally:
info("Shutting NGAS server down now")
sudo("service ngas-server stop ")
def start_APP_and_check_status():
"""
Starts the ngamsDaemon process and checks that the server is up and running.
Then it shuts down the server
"""
# We sleep 2 here as it was found on Mac deployment to docker container that the
# shell would exit before the ngasDaemon could detach, thus resulting in no startup.
virtualenv('ngamsDaemon start -cfg {0} && sleep 2'.format(env.tgt_cfg))
try:
res = virtualenv('ngamsDaemon status -cfg {0}'.format(env.tgt_cfg),
warn_only=True)
if res.failed:
failure(
"Couldn't contact NGAS server after starting it. "
"Check log files under %s/log/ to find out what went wrong" %
APP_source_dir(),
with_stars=False)
else:
success('NGAS server started correctly :)')
finally:
info("Shutting NGAS server down now")
virtualenv("ngamsDaemon stop -cfg {0}".format(env.tgt_cfg))
def prepare_APP_data_dir():
"""Creates a new APP root directory"""
info('Preparing {0} root directory'.format(env.APP_NAME))
nrd = APP_root_dir()
# tgt_cfg = os.path.join(nrd, 'cfg', 'ngamsServer.conf')
tgt_cfg = None
res = run('mkdir -p {0}'.format(nrd))
with cd(APP_source_dir()):
for d in env.APP_DATAFILES:
res = run('scp -r {0} {1}/.'.format(d, nrd), quiet=True)
if res.succeeded:
success("{0} data directory ready".format(env.APP_NAME))
return tgt_cfg
# Deal with the errors here
error = '{0} root directory preparation under {1} failed.\n'.format(
env.APP_NAME, nrd)
if res.return_code == 2:
error = (nrd + " already exists. Specify APP_OVERWRITE_ROOT to "
"overwrite, or a different APP_ROOT_DIR location")
else:
error = res
abort(error)