def install_klever_bridge_development(logger, src_dir):
logger.info('Install/update development Klever Bridge')
services = ('klever-bridge-development', 'klever-celery-development', 'klever-celerybeat-development')
stop_services(logger, services)
with Cd(os.path.join(src_dir, 'bridge')):
_install_klever_bridge(logger)
start_services(logger, services)
def install_klever_bridge_production(logger,
src_dir,
deploy_dir,
populate_just_production_presets=True):
logger.info('Install/update production Klever Bridge')
services = ('nginx', 'klever-bridge', 'klever-celery', 'klever-celerybeat')
stop_services(logger, services)
logger.info('Copy Klever Bridge configuration file for NGINX')
copy_from = os.path.join(src_dir, 'bridge/conf/nginx')
if os.path.exists('/etc/nginx/sites-enabled'):
shutil.copy(copy_from, '/etc/nginx/sites-enabled/klever-bridge.conf')
else:
shutil.copy(copy_from, '/etc/nginx/conf.d/klever-bridge.conf')
logger.info('Install/update Klever Bridge source/binary code')
shutil.rmtree('/var/www/klever-bridge', ignore_errors=True)
shutil.copytree(os.path.join(src_dir, 'bridge'),
'/var/www/klever-bridge/bridge')
shutil.copytree(os.path.join(src_dir, 'presets'),
'/var/www/klever-bridge/presets')
logger.info('Prepare media directory')
media = '/var/www/klever-bridge/bridge/media'
media_real = os.path.join(os.path.realpath(deploy_dir), 'klever-media')
shutil.rmtree(media)
execute_cmd(logger, 'mkdir', '-p', media_real)
execute_cmd(logger, 'ln', '-s', '-T', media_real, media)
with Cd('/var/www/klever-bridge/bridge'):
with open('bridge/settings.py', 'w') as fp:
fp.write('from bridge.{0} import *\n'.format('production'))
if not populate_just_production_presets:
fp.write('POPULATE_JUST_PRODUCTION_PRESETS = False\n')
_install_klever_bridge(logger)
logger.info('Collect static files')
execute_cmd(logger, sys.executable, './manage.py', 'collectstatic',
'--noinput')
# Make available data from media, logs and static for its actual user.
media_user = get_media_user(logger)
user_group = '{}:{}'.format(media_user, media_user)
execute_cmd(logger, 'chown', '-R', user_group, media_real)
execute_cmd(logger, 'chown', '-R', user_group,
'/var/www/klever-bridge/bridge/logs')
execute_cmd(logger, 'chown', '-R', user_group,
'/var/www/klever-bridge/bridge/static')
start_services(logger, services)
def _pre_uninstall(self, mode_services):
services = list(mode_services)
services.extend(
('klever-controller', 'klever-native-scheduler', 'klever-cgroup'))
if need_verifiercloud_scheduler(self.prev_deploy_info):
services.append('klever-verifiercloud-scheduler')
stop_services(self.logger, services, ignore_errors=True)
self.logger.info('Uninstall systemd services')
for dirpath, _, filenames in os.walk('/etc/systemd/system'):
for filename in filenames:
if filename.startswith('klever'):
service = os.path.join(dirpath, filename)
self.logger.info('Remove "{0}"'.format(service))
os.remove(service)
klever_env_file = '/etc/default/klever'
if os.path.exists(klever_env_file):
self.logger.info('Remove "{0}"'.format(klever_env_file))
os.remove(klever_env_file)
# Remove bridge files
bridge_path = os.path.join(self.args.deployment_directory,
'klever/bridge/bridge')
for path in ('settings.py', 'db.json', 'rmq.json'):
path = os.path.join(bridge_path, path)
if os.path.exists(path):
self.logger.info('Remove "{0}"'.format(path))
os.remove(path)
# Removing individual directories and files rather than the whole deployment directory allows to use standard
# locations like "/", "/usr" or "/usr/local" for deploying Klever.
for path in ('klever', 'klever-addons', 'klever-conf', 'klever-work',
'klever-media', 'klever.json'):
path = os.path.join(self.args.deployment_directory, path)
if os.path.exists(path) or os.path.islink(path):
self.logger.info('Remove "{0}"'.format(path))
if os.path.islink(path) or os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path)
# Remove deployment directory if it is empty
if os.path.exists(self.args.deployment_directory) and not os.listdir(
self.args.deployment_directory):
self.logger.info('Remove "{0}"'.format(
self.args.deployment_directory))
os.rmdir(self.args.deployment_directory)
# Remove Klever Bridge NGINX configuration if so.
for klever_bridge_nginx_conf_file in (
'/etc/nginx/sites-enabled/klever-bridge.conf',
'/etc/nginx/conf.d/klever-bridge.conf'):
if os.path.exists(klever_bridge_nginx_conf_file):
self.logger.info(
'Remove "{0}"'.format(klever_bridge_nginx_conf_file))
os.remove(klever_bridge_nginx_conf_file)
stop_services(self.logger, ('nginx', ))
start_services(self.logger, ('nginx', ))
try:
pwd.getpwnam('postgres')
except KeyError:
# Do nothing if user "postgres" does not exist.
pass
else:
self.logger.info('Delete PostgreSQL database')
execute_cmd(self.logger,
'dropdb',
'--if-exists',
'klever',
username='******')
self.logger.info('Delete PostgreSQL user')
execute_cmd(self.logger,
'psql',
'-c',
"DROP USER IF EXISTS klever",
username='******')
try:
pwd.getpwnam('klever')
except KeyError:
# Do nothing if user "klever" does not exist.
pass
else:
self.logger.info('Delete user "klever"')
execute_cmd(self.logger, 'userdel', 'klever')
try:
self.logger.info('Delete RabbitMQ user')
execute_cmd(self.logger, 'rabbitmqctl', 'delete_user', 'service')
except (FileNotFoundError, subprocess.CalledProcessError):
pass
# Try to remove httpd_t from the list of permissive domains.
try:
execute_cmd(self.logger, 'semanage', 'permissive', '-d', 'httpd_t')
except Exception:
pass
def configure_controller_and_schedulers(logger, development, src_dir,
deploy_dir, prev_deploy_info):
logger.info(
'(Re)configure {0} Klever Controller and Klever schedulers'.format(
'development' if development else 'production'))
services = [
'klever-controller', 'klever-native-scheduler', 'klever-cgroup'
]
if need_verifiercloud_scheduler(prev_deploy_info):
services.append('klever-verifiercloud-scheduler')
stop_services(logger, services)
conf_dir = os.path.join(src_dir, 'klever', 'scheduler', 'conf')
deploy_dir_abs = os.path.realpath(deploy_dir)
logger.info('Configure Klever Controller')
with open(os.path.join(conf_dir, 'controller.json')) as fp:
controller_conf = json.load(fp)
controller_conf['common']['working directory'] = os.path.join(
deploy_dir_abs, 'klever-work/controller')
controller_conf['Klever Bridge'].update({
'user': '******',
'password': '******'
})
controller_conf['client-controller']['consul'] = get_klever_addon_abs_path(
deploy_dir, prev_deploy_info, 'Consul')
with open(os.path.join(deploy_dir, 'klever-conf/controller.json'),
'w') as fp:
json.dump(controller_conf, fp, sort_keys=True, indent=4)
logger.info('Configure Klever Native Scheduler')
with open(os.path.join(conf_dir, 'native-scheduler.json')) as fp:
native_scheduler_conf = json.load(fp)
native_scheduler_conf['common']['working directory'] = os.path.join(
deploy_dir_abs, 'klever-work/native-scheduler')
if development:
native_scheduler_conf['common']['keep working directory'] = True
native_scheduler_conf['Klever Bridge'].update({
'user': '******',
'password': '******'
})
native_scheduler_conf['Klever jobs and tasks queue'].update({
'username':
'******',
'password':
'******'
})
native_scheduler_conf['scheduler'].update({
'disable CPU cores account':
True,
'job client configuration':
os.path.realpath(
os.path.join(deploy_dir,
'klever-conf/native-scheduler-job-client.json')),
'task client configuration':
os.path.realpath(
os.path.join(deploy_dir,
'klever-conf/native-scheduler-task-client.json'))
})
if development:
native_scheduler_conf['scheduler']['keep working directory'] = True
with open(os.path.join(deploy_dir, 'klever-conf/native-scheduler.json'),
'w') as fp:
json.dump(native_scheduler_conf, fp, sort_keys=True, indent=4)
logger.info('Configure Klever Native Scheduler Job Worker')
with open(os.path.join(conf_dir, 'job-client.json')) as fp:
job_client_conf = json.load(fp)
configure_task_and_job_configuration_paths(deploy_dir, prev_deploy_info,
job_client_conf)
with open(
os.path.join(deploy_dir,
'klever-conf/native-scheduler-job-client.json'),
'w') as fp:
json.dump(job_client_conf, fp, sort_keys=True, indent=4)
if need_verifiercloud_scheduler(prev_deploy_info):
logger.info('Configure Klever VerifierCloud Scheduler')
with open(os.path.join(conf_dir,
'verifiercloud-scheduler.json')) as fp:
verifiercloud_scheduler_conf = json.load(fp)
verifiercloud_scheduler_conf['common']['working directory'] = \
os.path.join(deploy_dir_abs, 'klever-work/verifiercloud-scheduler')
verifiercloud_scheduler_conf['Klever Bridge'].update({
'user':
'******',
'password':
'******'
})
verifiercloud_scheduler_conf['Klever jobs and tasks queue'].update({
'username':
'******',
'password':
'******'
})
verifiercloud_scheduler_conf['scheduler']['web client location'] =\
get_klever_addon_abs_path(deploy_dir, prev_deploy_info, 'VerifierCloud Client')
with open(
os.path.join(deploy_dir,
'klever-conf/verifiercloud-scheduler.json'),
'w') as fp:
json.dump(verifiercloud_scheduler_conf,
fp,
sort_keys=True,
indent=4)
logger.info('Configure Klever Native Scheduler Task Worker')
with open(os.path.join(conf_dir, 'task-client.json')) as fp:
task_client_conf = json.load(fp)
configure_task_and_job_configuration_paths(deploy_dir, prev_deploy_info,
task_client_conf)
verification_backends = task_client_conf['client'][
'verification tools'] = {}
for name, desc in prev_deploy_info['Klever Addons'][
'Verification Backends'].items():
if desc['name'] not in verification_backends:
verification_backends[desc['name']] = {}
verification_backends[desc['name']][desc['version']] = \
get_klever_addon_abs_path(deploy_dir, prev_deploy_info, name, verification_backend=True)
with open(
os.path.join(deploy_dir,
'klever-conf/native-scheduler-task-client.json'),
'w') as fp:
json.dump(task_client_conf, fp, sort_keys=True, indent=4)
start_services(logger, services)
