def _pre_install(self):
if self.prev_deploy_info:
self.logger.error(
'There is information on previous deployment (perhaps you try to install Klever second time)'
)
sys.exit(errno.EINVAL)
with open(self.args.deployment_configuration_file) as fp:
self.deploy_conf = json.load(fp)
self.logger.info('Create deployment directory')
os.makedirs(self.args.deployment_directory, exist_ok=True)
self.logger.info('Install init.d scripts')
for dirpath, _, filenames in os.walk(
os.path.join(os.path.dirname(__file__), os.path.pardir,
os.path.pardir, 'init.d')):
for filename in filenames:
shutil.copy(os.path.join(dirpath, filename),
os.path.join('/etc/init.d', filename))
execute_cmd(self.logger, 'update-rc.d', filename, 'defaults')
with open('/etc/default/klever', 'w') as fp:
fp.write('KLEVER_DEPLOYMENT_DIRECTORY="{0}"\n'.format(
os.path.realpath(self.args.deployment_directory)))
fp.write('KLEVER_DATA_DIR="{0}"\n'.format(
os.path.join(os.path.realpath(self.args.deployment_directory),
'klever', 'build bases')))
self._install_or_update_deps()
prepare_env(self.logger, self.args.deployment_directory)
self._pre_install_or_update()
def install_klever_bridge_development(logger, deploy_dir):
logger.info('Install/update development Klever Bridge')
services = ('klever-bridge-development', )
stop_services(logger, services)
# Do not overwrite directory "media" from sumbolically linked Git repository. Otherwise it will notice changes.
if not os.path.islink(os.path.join(deploy_dir, 'klever')):
logger.info('Prepare media directory')
media = os.path.join(deploy_dir, 'klever/bridge/media')
media_real = os.path.join(os.path.realpath(deploy_dir), 'klever-media')
if os.path.islink(media):
os.remove(media)
else:
shutil.rmtree(media)
execute_cmd(logger, 'mkdir', '-p', media_real)
execute_cmd(logger, 'ln', '-s', '-T', media_real, media)
with Cd(os.path.join(deploy_dir, 'klever/bridge')):
with open('bridge/settings.py', 'w') as fp:
fp.write('from bridge.{0} import *\n'.format('development'))
_install_klever_bridge(logger)
start_services(logger, services)
def open_shell(self):
self.logger.info(
'Open interactive SSH to instance "{0}" (IP: {1})'.format(
self.name, self.floating_ip))
execute_cmd(self.logger, 'ssh', '-o', 'StrictHostKeyChecking=no', '-i',
self.args.ssh_rsa_private_key_file,
'{0}@{1}'.format(self.args.ssh_username, self.floating_ip))
def _install_klever_bridge(logger):
logger.info('Configure Klever Bridge')
with open('bridge/db.json', 'w') as fp:
json.dump(
{
'ENGINE': 'django.db.backends.postgresql',
'HOST': '127.0.0.1',
'NAME': 'klever',
'USER': '******',
'PASSWORD': '******'
},
fp,
sort_keys=True,
indent=4)
logger.info('Update translations')
execute_cmd(logger, './manage.py', 'compilemessages')
logger.info('Migrate database')
execute_cmd(logger, './manage.py', 'migrate')
logger.info('Populate database')
execute_cmd(logger, './manage.py', 'PopulateUsers', '--exist-ok',
'--admin', '{"username": "******", "password": "******"}',
'--manager', '{"username": "******", "password": "******"}',
'--service', '{"username": "******", "password": "******"}')
execute_cmd(logger, './manage.py', 'Population')
def _pre_uninstall(self, mode_services):
services = list(mode_services)
services.extend(('klever-controller', 'klever-native-scheduler'))
if need_verifiercloud_scheduler(self.prev_deploy_info):
services.append('klever-verifiercloud-scheduler')
stop_services(self.logger, services, ignore_errors=True)
self.logger.info('Uninstall init.d scripts')
for dirpath, _, filenames in os.walk('/etc/init.d'):
for filename in filenames:
if filename.startswith('klever'):
execute_cmd(self.logger, 'update-rc.d', filename,
'disable')
os.unlink(os.path.join('/etc/init.d', filename))
if os.path.exists('/etc/default/klever'):
os.unlink('/etc/default/klever')
# Removing individual directories and files rather than the whole deployment directory allows to use standard
# locations like "/", "/usr" or "/usr/local" for deploying Klever.
for path in ('klever', 'klever-addons', 'klever-conf', 'klever-work',
'klever-media', 'klever.json'):
path = os.path.join(self.args.deployment_directory, path)
if os.path.exists(path) or os.path.islink(path):
self.logger.info('Remove "{0}"'.format(path))
if os.path.islink(path) or os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path)
try:
pwd.getpwnam('postgres')
except KeyError:
# Do nothing if user "postgres" does not exist.
pass
else:
self.logger.info('Drop PostgreSQL database')
execute_cmd(self.logger,
'dropdb',
'--if-exists',
'klever',
username='******')
self.logger.info('Drop PostgreSQL user')
execute_cmd(self.logger,
'psql',
'-c',
"DROP USER IF EXISTS klever",
username='******')
try:
pwd.getpwnam('klever')
except KeyError:
# Do nothing if user "klever" does not exist.
pass
else:
execute_cmd(self.logger, 'userdel', 'klever')
def install_fn(src, dst, allow_symlink=False, ignore=None):
if ignore and allow_symlink:
self.logger.error(
'You can not both use symbolic links and ignore some directories'
)
sys.exit(errno.EINVAL)
self.logger.info('Install "{0}" to "{1}"'.format(src, dst))
os.makedirs(dst if os.path.isdir(dst) else os.path.dirname(dst),
exist_ok=True)
if allow_symlink and self.args.allow_symbolic_links:
execute_cmd(self.logger, 'ln', '-s', src, dst)
else:
if os.path.isdir(src):
shutil.copytree(src,
dst,
symlinks=True,
ignore=lambda source, names: ignore or [])
else:
shutil.copy(src, dst)
def install_klever_bridge_production(logger,
deploy_dir,
populate_just_production_presets=True):
logger.info('Install/update production Klever Bridge')
services = ('nginx', 'klever-bridge')
stop_services(logger, services)
logger.info('Copy Klever Bridge configuration file for NGINX')
shutil.copy(os.path.join(deploy_dir, 'klever/bridge/conf/debian-nginx'),
'/etc/nginx/sites-enabled/klever-bridge')
logger.info('Update Klever Bridge source/binary code')
shutil.rmtree('/var/www/klever-bridge', ignore_errors=True)
shutil.copytree(os.path.join(deploy_dir, 'klever/bridge'),
'/var/www/klever-bridge')
logger.info('Prepare media directory')
media = '/var/www/klever-bridge/media'
media_real = os.path.join(os.path.realpath(deploy_dir), 'klever-media')
shutil.rmtree(media)
execute_cmd(logger, 'mkdir', '-p', media_real)
execute_cmd(logger, 'ln', '-s', '-T', media_real, media)
with Cd('/var/www/klever-bridge'):
with open('bridge/settings.py', 'w') as fp:
fp.write('from bridge.{0} import *\n'.format('production'))
if not populate_just_production_presets:
fp.write('POPULATE_JUST_PRODUCTION_PRESETS = False\n')
_install_klever_bridge(logger)
logger.info('Collect static files')
execute_cmd(logger, './manage.py', 'collectstatic', '--noinput')
# Make available data from media for its actual user.
execute_cmd(logger, 'chown', '-R', 'www-data:www-data', media_real)
start_services(logger, services)
def prepare_env(logger, deploy_dir):
logger.info('Create user "klever"')
execute_cmd(logger, 'useradd', 'klever')
logger.info('Prepare configurations directory')
execute_cmd(logger, 'mkdir', os.path.join(deploy_dir, 'klever-conf'))
logger.info('Prepare working directory')
work_dir = os.path.join(deploy_dir, 'klever-work')
execute_cmd(logger, 'mkdir', work_dir)
execute_cmd(logger, 'chown', '-LR', 'klever', work_dir)
openssl_header = '/usr/include/openssl/opensslconf.h'
if not os.path.exists(openssl_header):
logger.info(
'Create soft links for libssl to build new versions of the Linux kernel'
)
execute_cmd(logger, 'ln', '-s',
'/usr/include/x86_64-linux-gnu/openssl/opensslconf.h',
openssl_header)
crts = glob.glob('/usr/lib/x86_64-linux-gnu/crt*.o')
args = []
for crt in crts:
if not os.path.exists(os.path.join('/usr/lib', os.path.basename(crt))):
args.append(crt)
if args:
logger.info('Prepare CIF environment')
args.append('/usr/lib')
execute_cmd(logger, 'ln', '-s', *args)
logger.info('Create PostgreSQL user')
execute_cmd(logger,
'psql',
'-c',
"CREATE USER klever WITH CREATEDB PASSWORD 'klever'",
username='******')
logger.info('Create PostgreSQL database')
execute_cmd(logger,
'createdb',
'-T',
'template0',
'-E',
'utf8',
'klever',
username='******')
def cmd_fn(*args):
execute_cmd(self.logger, *args)
def install_deps(logger, deploy_conf, prev_deploy_info, non_interactive, update_pckgs, update_py_pckgs):
if non_interactive:
# Do not require users input.
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
# Get packages to be installed/updated.
pckgs_to_install = []
pckgs_to_update = []
py_pckgs_to_install = []
py_pckgs_to_update = []
def get_pckgs(pckgs):
_pckgs = []
for val in pckgs.values():
_pckgs.extend(val)
return _pckgs
# We can skip installation/update of dependencies if nothing is specified, but most likely one prepares
# deployment configuration file incorrectly.
if 'Packages' not in deploy_conf:
logger.error('Deployment configuration file does not describe packages to be installed/updated')
sys.exit(errno.EINVAL)
if 'Python3 Packages' not in deploy_conf:
logger.error('Deployment configuration file does not describe Python3 packages to be installed/updated')
sys.exit(errno.EINVAL)
new_pckgs = get_pckgs(deploy_conf['Packages'])
new_py_pckgs = get_pckgs(deploy_conf['Python3 Packages'])
if 'Packages' in prev_deploy_info:
for pckg in new_pckgs:
if pckg in prev_deploy_info['Packages']:
pckgs_to_update.append(pckg)
else:
pckgs_to_install.append(pckg)
else:
# All packages should be installed.
pckgs_to_install = new_pckgs
if 'Python3 Packages' in prev_deploy_info:
for py_pckg in new_py_pckgs:
if py_pckg in prev_deploy_info['Python3 Packages']:
py_pckgs_to_update.append(py_pckg)
else:
py_pckgs_to_install.append(py_pckg)
else:
# All Python3 packages should be installed.
py_pckgs_to_install = new_py_pckgs
if pckgs_to_install or (pckgs_to_update and update_pckgs):
logger.info('Update packages list')
execute_cmd(logger, 'apt-get', 'update')
if pckgs_to_install:
logger.info('Install packages:\n {0}'.format('\n '.join(pckgs_to_install)))
args = ['apt-get', 'install']
if non_interactive:
args.append('--assume-yes')
args.extend(pckgs_to_install)
execute_cmd(logger, *args)
# Remember what packages were installed just if everything went well.
if 'Packages' not in prev_deploy_info:
prev_deploy_info['Packages'] = []
prev_deploy_info['Packages'] = sorted(prev_deploy_info['Packages'] + pckgs_to_install)
if py_pckgs_to_install:
logger.info('Install Python3 packages:\n {0}'.format('\n '.join(py_pckgs_to_install)))
execute_cmd(logger, 'pip3', 'install', *py_pckgs_to_install)
# Remember what Python3 packages were installed just if everything went well.
if 'Python3 Packages' not in prev_deploy_info:
prev_deploy_info['Python3 Packages'] = []
prev_deploy_info['Python3 Packages'] = sorted(prev_deploy_info['Python3 Packages'] + py_pckgs_to_install)
if pckgs_to_update and update_pckgs:
logger.info('Update packages:\n {0}'.format('\n '.join(pckgs_to_update)))
args = ['apt-get', 'upgrade']
if non_interactive:
args.append('--assume-yes')
args.extend(pckgs_to_update)
execute_cmd(logger, *args)
if py_pckgs_to_update and update_py_pckgs:
logger.info('Update Python3 packages:\n {0}'.format('\n '.join(py_pckgs_to_update)))
execute_cmd(logger, 'pip3', 'install', '--upgrade', *py_pckgs_to_update)