def __enter__(self):
self.__copy_deployment_files()
self.logger.info('Copy sources that can be used during {0}'.format(
self.action))
with Cd(self.args.source_directory):
try:
klever_copy = os.path.join(tempfile.mkdtemp(), 'klever')
execute_cmd(self.logger, 'git', 'clone', '.', klever_copy)
# Store Klever version to dedicated file and remove directory ".git" since it occupies too much space.
with open(os.path.join(klever_copy, 'version'), 'w') as fp:
fp.write(setuptools_scm.get_version())
execute_cmd(self.logger, 'rm', '-rf', klever_copy + '/.git')
# Development Klever Bridge runs from Klever sources and it creates directories like __pycache__, media,
# etc. with root access. We need to backup media and to restore it after update of Klever sources. Other
# directories are out of interest, but they should not hinder rsync.
sftp = self.ssh.ssh.open_sftp()
media_exists = False
try:
sftp.stat('klever/bridge/media')
media_exists = True
self.ssh.execute_cmd('mv klever/bridge/media media-backup')
self.ssh.execute_cmd('sudo rm -rf klever/bridge')
except IOError:
pass
self.ssh.rsync(klever_copy, '~/')
if media_exists:
self.ssh.execute_cmd('sudo rm -rf klever/bridge/media')
self.ssh.execute_cmd('mv media-backup klever/bridge/media')
finally:
if os.path.exists(klever_copy):
shutil.rmtree(klever_copy)
def _pre_install(self):
if os.path.exists(self.prev_deploy_info_file) and not self.keep_addons_and_build_bases:
self.logger.error(
'There is information on previous deployment (perhaps you try to install Klever second time)')
sys.exit(errno.EINVAL)
with open(self.args.deployment_configuration_file) as fp:
self.deploy_conf = json.load(fp)
self.logger.info('Create deployment directory')
os.makedirs(self.args.deployment_directory, exist_ok=True)
with open('/etc/default/klever', 'w') as fp:
fp.write('KLEVER_SOURCE_DIRECTORY="{0}"\n'.format(os.path.realpath(self.args.source_directory)))
fp.write('KLEVER_DEPLOYMENT_DIRECTORY="{0}"\n'.format(os.path.realpath(self.args.deployment_directory)))
fp.write('KLEVER_DATA_DIR="{0}"\n'
.format(os.path.realpath(self.args.data_directory)
if self.args.data_directory
else os.path.join(os.path.realpath(self.args.deployment_directory), 'build bases')))
fp.write("KLEVER_WORKERS={}\n".format(os.cpu_count() + 1))
fp.write("KLEVER_PYTHON_BIN_DIR={}\n".format(os.path.dirname(sys.executable)))
fp.write("KLEVER_PYTHON={}\n".format(sys.executable))
media_user = get_media_user(self.logger)
self.logger.info('Install systemd configuration files and services')
execute_cmd(self.logger, 'mkdir', '-p', '/etc/conf.d')
for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir,
'systemd', 'conf.d')):
for filename in filenames:
shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/conf.d', filename))
for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir,
'systemd', 'tmpfiles.d')):
for filename in filenames:
shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/tmpfiles.d', filename))
replace_media_user(os.path.join('/etc/tmpfiles.d', filename), media_user)
for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir,
'systemd', 'system')):
for filename in filenames:
shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/systemd/system', filename))
replace_media_user(os.path.join('/etc/systemd/system', filename), media_user)
self._install_or_update_deps()
prepare_env(self.logger, self.args.deployment_directory)
self._pre_install_or_update()
# Set environment variable JAVA to point out absolute path to java executable to be used for executing Java
# programs within Klever. At the moment the same java will be used for all Java programs but that may be changed
# in future.
with open('/etc/default/klever', 'a+') as fp:
fp.write("JAVA={}\n".format(
os.path.join(os.path.realpath(self.args.deployment_directory), 'klever-addons', 'JRE',
self.prev_deploy_info['Klever Addons']['JRE']['executable path'], 'java')))
def open_shell(self):
self.logger.info(
'Open interactive SSH to instance "{0}" (IP: {1})'.format(
self.name, self.floating_ip))
execute_cmd(self.logger,
'ssh',
'-o',
'StrictHostKeyChecking=no',
'-i',
self.args.ssh_rsa_private_key_file,
'{0}@{1}'.format(self.args.ssh_username, self.floating_ip),
keep_stdout=True)
def __get_build_base_version(self, klever_build_base):
version = self.deploy_conf['Klever Build Bases'][klever_build_base].get('version')
if version:
return version
klever_build_base_path = self.deploy_conf['Klever Build Bases'][klever_build_base]['path']
if os.path.isfile(klever_build_base_path):
# Use md5 checksum of the archive as version
output = execute_cmd(self.logger, 'md5sum', klever_build_base_path, stderr=subprocess.DEVNULL,
get_output=True).rstrip()
version = output.split(' ')[0]
elif os.path.isdir(klever_build_base_path):
# Use unique identifier of the build base as version
try:
version = Clade(klever_build_base_path).get_uuid()
except RuntimeError:
self.logger.error(f'"{klever_build_base}" is not a valid Clade build base')
sys.exit(errno.EINVAL)
else:
# Otherwise build base is probably a link to the remote file
# Our build bases are mostly stored at redmine, which has unique links
# Here we use this link as version
version = klever_build_base_path
return version
def __enter__(self):
self.__copy_deployment_files()
self.logger.info('Copy sources that can be used during {0}'.format(self.action))
with Cd(self.args.source_directory):
try:
klever_copy = os.path.join(tempfile.mkdtemp(), 'klever')
execute_cmd(self.logger, 'git', 'clone', '.', klever_copy)
# Store Klever version to dedicated file and remove directory ".git" since it occupies too much space.
with open(os.path.join(klever_copy, 'version'), 'w') as fp:
fp.write(setuptools_scm.get_version())
execute_cmd(self.logger, 'rm', '-rf', klever_copy + '/.git')
self.ssh.rsync(klever_copy, '~/')
finally:
if os.path.exists(klever_copy):
shutil.rmtree(klever_copy)
def _install_fn(self, src, dst, allow_symlink=False, ignore=None):
if ignore and allow_symlink:
self.logger.error('You can not both use symbolic links and ignore some directories')
sys.exit(errno.EINVAL)
self.logger.info('Install "{0}" to "{1}"'.format(src, dst))
os.makedirs(dst if os.path.isdir(dst) else os.path.dirname(dst), exist_ok=True)
if allow_symlink and self.args.allow_symbolic_links:
execute_cmd(self.logger, 'ln', '-s', src, dst)
else:
if os.path.isdir(src):
shutil.copytree(src, dst, symlinks=True, ignore=lambda source, names: ignore or [])
else:
shutil.copy(src, dst)
def rsync(self, host_path, instance_path):
if instance_path:
self.execute_cmd(f'mkdir -p {instance_path}')
else:
instance_path = "~/"
# stderr=subprocess.DEVNULL is required to suppress WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!
# maybe there is a better way to fix it
execute_cmd(
self.logger,
'rsync', '-ar',
'-e', f'ssh -o StrictHostKeyChecking=no -i {self.args.ssh_rsa_private_key_file}',
host_path,
f'{OS_USER}@{self.floating_ip}:{instance_path}',
stderr=subprocess.DEVNULL
)
def _pre_install(self):
if self.prev_deploy_info:
self.logger.error(
'There is information on previous deployment (perhaps you try to install Klever second time)')
sys.exit(errno.EINVAL)
with open(self.args.deployment_configuration_file) as fp:
self.deploy_conf = json.load(fp)
self.logger.info('Create deployment directory')
os.makedirs(self.args.deployment_directory, exist_ok=True)
with open('/etc/default/klever', 'w') as fp:
fp.write('KLEVER_SOURCE_DIRECTORY="{0}"\n'.format(os.path.realpath(self.args.source_directory)))
fp.write('KLEVER_DEPLOYMENT_DIRECTORY="{0}"\n'.format(os.path.realpath(self.args.deployment_directory)))
fp.write('KLEVER_DATA_DIR="{0}"\n'
.format(os.path.join(os.path.realpath(self.args.deployment_directory), 'klever', 'build bases')
if len(self.deploy_conf['Klever Build Bases'])
else os.path.join(os.path.realpath(self.args.source_directory), 'build bases')))
fp.write("KLEVER_WORKERS={}\n".format(os.cpu_count() + 1))
fp.write("KLEVER_PYTHON_BIN_DIR={}\n".format(os.path.dirname(sys.executable)))
fp.write("KLEVER_PYTHON={}\n".format(sys.executable))
media_user = get_media_user(self.logger)
self.logger.info('Install systemd configuration files and services')
execute_cmd(self.logger, 'mkdir', '-p', '/etc/conf.d')
for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir,
'systemd', 'conf.d')):
for filename in filenames:
shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/conf.d', filename))
for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir,
'systemd', 'tmpfiles.d')):
for filename in filenames:
shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/tmpfiles.d', filename))
replace_media_user(os.path.join('/etc/tmpfiles.d', filename), media_user)
for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir,
'systemd', 'system')):
for filename in filenames:
shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/systemd/system', filename))
replace_media_user(os.path.join('/etc/systemd/system', filename), media_user)
self._install_or_update_deps()
prepare_env(self.logger, self.args.deployment_directory)
self._pre_install_or_update()
def __enter__(self):
self.logger.info('Copy deployment configuration file')
self.ssh.sftp_put(self.args.deployment_configuration_file, 'klever.json')
self.logger.info('Copy sources that can be used during {0}'.format(self.action))
with Cd(self.args.source_directory):
try:
execute_cmd(self.logger, 'git', 'clone', '.', '__klever')
execute_cmd(self.logger, 'tar', '-C', '__klever', '-cf', '__klever.tar.gz', '.')
self.ssh.sftp_put('__klever.tar.gz', 'klever/klever.tar.gz')
self.ssh.execute_cmd('tar --warning no-unknown-keyword -C klever -xf klever/klever.tar.gz')
self.ssh.execute_cmd('rm klever/klever.tar.gz')
finally:
if os.path.exists('__klever'):
shutil.rmtree('__klever')
if os.path.exists('__klever.tar.gz'):
os.remove('__klever.tar.gz')
def rsync(self, host_path, instance_path):
if not instance_path:
instance_path = "~/"
if instance_path.startswith('~'):
# with '-s' rsync sends all filenames without allowing the remote shell to interpret them
# so, we need to explicitly expand ~ here
instance_path = instance_path.replace('~', f'/home/{OS_USER}', 1)
self.logger.debug(
'Execute rsync command to instance "{}" (IP: {})\ncopy {} to {}'.
format(self.name, self.floating_ip, host_path, instance_path))
# mkdir also doesn't work with paths inside quotes that contain ~
self.execute_cmd(f'mkdir -p "{instance_path}"')
if os.path.isfile(host_path) and (tarfile.is_tarfile(host_path)
or zipfile.is_zipfile(host_path)):
rsync_flags = '-as'
else:
# with '-z' rsync compresses the transmitted data
rsync_flags = '-asz'
# stderr=subprocess.DEVNULL is required to suppress WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED!
# maybe there is a better way to fix it
execute_cmd(
self.logger,
'rsync',
rsync_flags,
'--del',
'-e',
f'ssh -o StrictHostKeyChecking=no -i {self.args.ssh_rsa_private_key_file}',
# Exclude Python build directories since rsync can fail to remove them. Indeed, this is a workaround that is
# necessary due to we incorrectly install Klever on OpenStack instances using the superuser rather than the
# virtual environment.
'--exclude',
'build',
'--exclude',
'klever.egg-info',
host_path,
f'{OS_USER}@{self.floating_ip}:{instance_path}',
stderr=subprocess.DEVNULL)
def __enter__(self):
self.logger.info('Copy deployment configuration file')
self.ssh.sftp_put(self.args.deployment_configuration_file,
'klever.json')
self.logger.info('Copy sources that can be used during {0}'.format(
self.action))
with Cd(self.args.source_directory):
try:
execute_cmd(self.logger, 'git', 'clone', '.', '__klever')
# Store Klever version to dedicated file and remove directory ".git" since it occupies too much space.
with Cd('__klever'):
version = get_version()
with open('version', 'w') as fp:
fp.write(version)
execute_cmd(self.logger, 'rm', '-rf', '__klever/.git')
execute_cmd(self.logger, 'tar', '-C', '__klever', '-cf',
'__klever.tar.gz', '.')
self.ssh.sftp_put('__klever.tar.gz', 'klever/klever.tar.gz')
self.ssh.execute_cmd(
'tar --warning no-unknown-keyword -C klever -xf klever/klever.tar.gz'
)
self.ssh.execute_cmd('rm klever/klever.tar.gz')
finally:
if os.path.exists('__klever'):
shutil.rmtree('__klever')
if os.path.exists('__klever.tar.gz'):
os.remove('__klever.tar.gz')
def prepare_env(logger, deploy_dir):
logger.info('Prepare environment')
try:
logger.debug('Try to create user "klever"')
execute_cmd(logger, 'useradd', 'klever')
except subprocess.CalledProcessError:
logger.debug('User "klever" already exists')
try:
logger.debug('Obtain execute access to {!r} home directory'.format(os.getlogin()))
execute_cmd(logger, 'chmod', 'o+x', os.path.join('/', 'home', os.getlogin()))
except Exception:
pass
logger.debug('Prepare configurations directory')
execute_cmd(logger, 'mkdir', os.path.join(deploy_dir, 'klever-conf'))
logger.debug('Prepare working directory')
work_dir = os.path.join(deploy_dir, 'klever-work')
execute_cmd(logger, 'mkdir', work_dir)
execute_cmd(logger, 'chown', '-LR', 'klever', work_dir)
openssl_header = '/usr/include/openssl/opensslconf.h'
if not os.path.exists(openssl_header):
logger.debug('Create soft links for libssl to build new versions of the Linux kernel')
execute_cmd(logger, 'ln', '-s', '/usr/include/x86_64-linux-gnu/openssl/opensslconf.h', openssl_header)
crts = glob.glob('/usr/lib/x86_64-linux-gnu/crt*.o')
args = []
for crt in crts:
if not os.path.exists(os.path.join('/usr/lib', os.path.basename(crt))):
args.append(crt)
if args:
logger.debug('Prepare CIF environment')
args.append('/usr/lib')
execute_cmd(logger, 'ln', '-s', *args)
logger.debug('Try to initialise PostgreSQL')
try:
execute_cmd(logger, 'postgresql-setup', '--initdb', '--unit', 'postgresql')
except FileNotFoundError:
# postgresql-setup may not be present in the system. On some systems like openSUSE it is necessary to start the
# PostgreSQL service at least once so that necessary initialization will be performed automatically.
execute_cmd(logger, 'service', 'postgresql', 'restart')
except subprocess.CalledProcessError:
# postgresql-setup may fail if it was already executed before
pass
# Search for pg_hba.conf in all possible locations
pg_hba_conf_file = None
for path in ('/etc/postgresql', '/var/lib/pgsql/data'):
try:
pg_hba_conf_file = execute_cmd(logger, 'find', path, '-name', 'pg_hba.conf', get_output=True).rstrip()
except subprocess.CalledProcessError:
continue
with open(pg_hba_conf_file) as fp:
pg_hba_conf = fp.readlines()
with open(pg_hba_conf_file, 'w') as fp:
for line in pg_hba_conf:
# change ident to md5
if line.split() == ['host', 'all', 'all', '127.0.0.1/32', 'ident']:
line = 'host all all 127.0.0.1/32 md5\n'
fp.write(line)
execute_cmd(logger, 'service', 'postgresql', 'restart')
if not pg_hba_conf_file:
logger.error('Could not find PostgreSQL configuration file')
sys.exit(errno.EINVAL)
logger.debug('Start and enable PostgreSQL service')
execute_cmd(logger, 'systemctl', 'start', 'postgresql')
execute_cmd(logger, 'systemctl', 'enable', 'postgresql')
logger.debug('Create PostgreSQL user')
execute_cmd(logger, 'psql', '-c', "CREATE USER klever WITH CREATEDB PASSWORD 'klever'", username='******')
logger.debug('Create PostgreSQL database')
execute_cmd(logger, 'createdb', '-T', 'template0', '-E', 'utf-8', 'klever', username='******')
logger.debug('Start and enable RabbitMQ server service')
execute_cmd(logger, 'systemctl', 'start', 'rabbitmq-server.service')
execute_cmd(logger, 'systemctl', 'enable', 'rabbitmq-server.service')
logger.debug('Create RabbitMQ user')
execute_cmd(logger, 'rabbitmqctl', 'add_user', 'service', 'service')
execute_cmd(logger, 'rabbitmqctl', 'set_user_tags', 'service', 'administrator')
execute_cmd(logger, 'rabbitmqctl', 'set_permissions', '-p', '/', 'service', '.*', '.*', '.*')
def install_klever_bridge_production(logger,
src_dir,
deploy_dir,
populate_just_production_presets=True,
update=False):
logger.info('Install/update production Klever Bridge')
services = ('nginx', 'klever-bridge', 'klever-celery', 'klever-celerybeat')
stop_services(logger, services)
logger.info('Copy Klever Bridge configuration file for NGINX')
copy_from = os.path.join(src_dir, 'bridge/conf/nginx')
if os.path.exists('/etc/nginx/sites-enabled'):
shutil.copy(copy_from, '/etc/nginx/sites-enabled/klever-bridge.conf')
else:
shutil.copy(copy_from, '/etc/nginx/conf.d/klever-bridge.conf')
logger.info('Install/update Klever Bridge source/binary code')
shutil.rmtree('/var/www/klever-bridge', ignore_errors=True)
shutil.copytree(os.path.join(src_dir, 'bridge'),
'/var/www/klever-bridge/bridge',
ignore=shutil.ignore_patterns('test_files'))
shutil.copytree(os.path.join(src_dir, 'presets'),
'/var/www/klever-bridge/presets')
logger.info('Prepare media directory')
media = '/var/www/klever-bridge/bridge/media'
media_real = os.path.realpath(
os.path.join(os.path.realpath(deploy_dir), 'klever-media'))
shutil.rmtree(media)
execute_cmd(logger, 'mkdir', '-p', media_real)
execute_cmd(logger, 'ln', '-s', '-T', media_real, media)
with Cd('/var/www/klever-bridge/bridge'):
with open('bridge/settings.py', 'w') as fp:
fp.write('from bridge.{0} import *\n'.format('production'))
if not populate_just_production_presets:
fp.write('POPULATE_JUST_PRODUCTION_PRESETS = False\n')
_install_klever_bridge(logger, update)
logger.info('Collect static files')
execute_cmd(logger, sys.executable, './manage.py', 'collectstatic',
'--noinput')
# Make available data from media, logs and static for its actual user.
media_user = get_media_user(logger)
user_group = '{}:{}'.format(media_user, media_user)
execute_cmd(logger, 'chown', '-R', user_group, media_real)
execute_cmd(logger, 'chown', '-R', user_group,
'/var/www/klever-bridge/bridge/logs')
execute_cmd(logger, 'chown', '-R', user_group,
'/var/www/klever-bridge/bridge/static')
# Try to add httpd_t to the list of permissive domains.
try:
execute_cmd(logger, 'semanage', 'permissive', '-a', 'httpd_t')
except Exception:
pass
start_services(logger, services)
def _install_klever_bridge(logger, update):
logger.info('Update translations')
execute_cmd(logger, sys.executable, './manage.py', 'compilemessages')
logger.info('Migrate database')
execute_cmd(logger, sys.executable, './manage.py', 'migrate')
logger.info('Populate database')
# We need to create users once. Otherwise this can overwrite their settings changed manually.
if not update:
execute_cmd(logger, sys.executable, './manage.py', 'createuser',
'--username', 'admin', '--password', 'admin', '--staff',
'--superuser')
execute_cmd(logger, sys.executable, './manage.py', 'createuser',
'--username', 'manager', '--password', 'manager', '--role',
'2')
execute_cmd(logger, sys.executable, './manage.py', 'createuser',
'--username', 'service', '--password', 'service', '--role',
'4')
execute_cmd(logger, sys.executable, './manage.py', 'populate', '--all')
def install_deps(logger, deploy_conf, prev_deploy_info, non_interactive,
update_pckgs):
if non_interactive:
# Do not require users input.
os.environ['DEBIAN_FRONTEND'] = 'noninteractive'
# Get packages to be installed/updated.
pckgs_to_install = []
pckgs_to_update = []
deploy_conf['Packages'] = load_deps_conf(logger).strip().split(' ')
if 'Packages' in prev_deploy_info:
for pckg in deploy_conf['Packages']:
if pckg in prev_deploy_info['Packages']:
pckgs_to_update.append(pckg)
else:
pckgs_to_install.append(pckg)
else:
# All packages should be installed.
pckgs_to_install = deploy_conf['Packages']
if pckgs_to_install or (pckgs_to_update and update_pckgs):
logger.info('Update packages list')
if shutil.which('apt'):
execute_cmd(logger, 'apt', 'update')
elif shutil.which('dnf'):
execute_cmd(logger, 'dnf', 'update')
elif shutil.which('zypper'):
execute_cmd(logger, 'zypper', 'ref')
elif shutil.which('yum'):
execute_cmd(logger, 'yum', 'check-update')
else:
logger.error('Your Linux distribution is not supported')
sys.exit(errno.EINVAL)
if pckgs_to_install:
logger.info('Install packages:\n {0}'.format(
'\n '.join(pckgs_to_install)))
for util in ('apt', 'dnf', 'zypper', 'yum'):
if shutil.which(util):
args = [util, 'install']
if non_interactive:
args.append('-y')
args.extend(pckgs_to_install)
execute_cmd(logger, *args)
break
else:
logger.error('Your Linux distribution is not supported')
sys.exit(errno.EINVAL)
# Remember what packages were installed just if everything went well.
if 'Packages' not in prev_deploy_info:
prev_deploy_info['Packages'] = []
prev_deploy_info['Packages'] = sorted(prev_deploy_info['Packages'] +
pckgs_to_install)
if pckgs_to_update and update_pckgs:
logger.info('Update packages:\n {0}'.format(
'\n '.join(pckgs_to_update)))
for util in ('apt', 'dnf', 'zypper', 'yum'):
if shutil.which(util):
if util in ('apt', 'dnf'):
args = [util, 'upgrade']
elif util in ('yum', 'zypper'):
args = [util, 'update']
if non_interactive:
args.append('-y')
args.extend(pckgs_to_install)
execute_cmd(logger, *args)
break
else:
raise RuntimeError('Your Linux distribution is not supported')
def _cmd_fn(self, *args):
execute_cmd(self.logger, *args)
def install(self):
self._pre_install()
execute_cmd(self.logger, 'systemd-tmpfiles', '--create')
self._install_or_update()
self._post_install_or_update(self._IS_DEV)
def _pre_uninstall(self, mode_services):
services = list(mode_services)
services.extend(
('klever-controller', 'klever-native-scheduler', 'klever-cgroup'))
if need_verifiercloud_scheduler(self.prev_deploy_info):
services.append('klever-verifiercloud-scheduler')
stop_services(self.logger, services, ignore_errors=True)
self.logger.info('Uninstall systemd services')
for dirpath, _, filenames in os.walk('/etc/systemd/system'):
for filename in filenames:
if filename.startswith('klever'):
service = os.path.join(dirpath, filename)
self.logger.info('Remove "{0}"'.format(service))
os.remove(service)
klever_env_file = '/etc/default/klever'
if os.path.exists(klever_env_file):
self.logger.info('Remove "{0}"'.format(klever_env_file))
os.remove(klever_env_file)
# Remove bridge files
bridge_path = os.path.join(self.args.deployment_directory,
'klever/bridge/bridge')
for path in ('settings.py', 'db.json', 'rmq.json'):
path = os.path.join(bridge_path, path)
if os.path.exists(path):
self.logger.info('Remove "{0}"'.format(path))
os.remove(path)
# Removing individual directories and files rather than the whole deployment directory allows to use standard
# locations like "/", "/usr" or "/usr/local" for deploying Klever.
for path in ('klever', 'klever-addons', 'klever-conf', 'klever-work',
'klever-media', 'klever.json'):
path = os.path.join(self.args.deployment_directory, path)
if os.path.exists(path) or os.path.islink(path):
self.logger.info('Remove "{0}"'.format(path))
if os.path.islink(path) or os.path.isfile(path):
os.remove(path)
else:
shutil.rmtree(path)
# Remove deployment directory if it is empty
if os.path.exists(self.args.deployment_directory) and not os.listdir(
self.args.deployment_directory):
self.logger.info('Remove "{0}"'.format(
self.args.deployment_directory))
os.rmdir(self.args.deployment_directory)
# Remove Klever Bridge NGINX configuration if so.
for klever_bridge_nginx_conf_file in (
'/etc/nginx/sites-enabled/klever-bridge.conf',
'/etc/nginx/conf.d/klever-bridge.conf'):
if os.path.exists(klever_bridge_nginx_conf_file):
self.logger.info(
'Remove "{0}"'.format(klever_bridge_nginx_conf_file))
os.remove(klever_bridge_nginx_conf_file)
stop_services(self.logger, ('nginx', ))
start_services(self.logger, ('nginx', ))
try:
pwd.getpwnam('postgres')
except KeyError:
# Do nothing if user "postgres" does not exist.
pass
else:
self.logger.info('Delete PostgreSQL database')
execute_cmd(self.logger,
'dropdb',
'--if-exists',
'klever',
username='******')
self.logger.info('Delete PostgreSQL user')
execute_cmd(self.logger,
'psql',
'-c',
"DROP USER IF EXISTS klever",
username='******')
try:
pwd.getpwnam('klever')
except KeyError:
# Do nothing if user "klever" does not exist.
pass
else:
self.logger.info('Delete user "klever"')
execute_cmd(self.logger, 'userdel', 'klever')
try:
self.logger.info('Delete RabbitMQ user')
execute_cmd(self.logger, 'rabbitmqctl', 'delete_user', 'service')
except (FileNotFoundError, subprocess.CalledProcessError):
pass
# Try to remove httpd_t from the list of permissive domains.
try:
execute_cmd(self.logger, 'semanage', 'permissive', '-d', 'httpd_t')
except Exception:
pass
def _install_klever_bridge(logger):
logger.info('Update translations')
execute_cmd(logger, sys.executable, './manage.py', 'compilemessages')
logger.info('Migrate database')
execute_cmd(logger, sys.executable, './manage.py', 'migrate')
logger.info('Populate database')
execute_cmd(logger, sys.executable, './manage.py', 'createuser',
'--username', 'admin', '--password', 'admin',
'--staff', '--superuser')
execute_cmd(logger, sys.executable, './manage.py', 'createuser',
'--username', 'manager', '--password', 'manager',
'--role', '2')
execute_cmd(logger, sys.executable, './manage.py', 'createuser',
'--username', 'service', '--password', 'service',
'--role', '4')
execute_cmd(logger, sys.executable, './manage.py', 'populate', '--all')
def _install_entity(self, name, src_dir, deploy_dir, deploy_conf, prev_deploy_info, build_base=False):
if name not in deploy_conf:
self.logger.error(f'"{name}" is not described')
sys.exit(errno.EINVAL)
deploy_dir = os.path.normpath(deploy_dir)
desc = deploy_conf[name]
if 'version' not in desc:
self.logger.error(f'Version is not specified for "{name}"')
sys.exit(errno.EINVAL)
version = desc['version']
if 'path' not in desc:
self.logger.error(f'Path is not specified for "{name}"')
sys.exit(errno.EINVAL)
path = desc['path']
o = urllib.parse.urlparse(path)
if not o[0]:
path = make_canonical_path(src_dir, path)
refs = {}
try:
ref_strs = execute_cmd(self.logger, 'git', 'ls-remote', '--refs', path, stderr=subprocess.DEVNULL,
get_output=True).rstrip().split('\n')
is_git_repo = True
for ref_str in ref_strs:
commit, ref = ref_str.split('\t')
refs[ref] = commit
except subprocess.CalledProcessError:
is_git_repo = False
if is_git_repo and version != 'CURRENT':
# Version can be either some reference or commit hash. In the former case we need to get corresponding
# commit hash since it can differ from the previous one installed before for the same reference. In the
# latter case we will fail below one day if commit hash isn't valid.
# Note that here we can use just Git commands working with remote repositories since we didn't clone them
# yet and we don't want do this if update isn't necessary.
for prefix in ('refs/heads/', 'refs/tags/'):
if prefix + version in refs:
version = refs[prefix + version]
break
prev_version = prev_deploy_info[name]['version'] if name in prev_deploy_info else None
if version == prev_version and version != 'CURRENT':
self.logger.info(f'"{name}" is up to date (version: "{version}")')
return False
entity_kind = "Klever build base" if build_base else "Klever addon"
if prev_version:
self.logger.info(f'Update {entity_kind} "{name}" from version "{prev_version}" to version "{version}"')
else:
self.logger.info(f'Install {entity_kind} "{name}" (version: "{version}")')
# Remove previous version of entity if so. Do not make this in depend on previous version since it can be unset
# while entity is deployed. For instance, this can be the case when entity deployment fails somewhere in the
# middle.
self._cmd_fn('rm', '-rf', deploy_dir)
# Install new version of entity.
tmp_file = None
tmp_dir = None
try:
instance_path = os.path.join(deploy_dir, os.path.basename(path))
# Clone remote Git repository.
if (o[0] == 'git' or is_git_repo) and not os.path.exists(path):
tmp_dir = tempfile.mkdtemp()
execute_cmd(self.logger, 'git', 'clone', '-q', '--recursive', path, tmp_dir)
path = tmp_dir
# Download remote file.
elif o[0] in ('http', 'https', 'ftp'):
_, tmp_file = tempfile.mkstemp()
execute_cmd(self.logger, 'wget', '-O', tmp_file, '-q', path)
path = tmp_file
elif o[0]:
self.logger.error(f'"{name}" is provided in unsupported form "{o[0]}"')
sys.exit(errno.EINVAL)
elif not os.path.exists(path):
self.logger.error(f'Path "{path}" does not exist')
sys.exit(errno.ENOENT)
if is_git_repo:
if version == 'CURRENT':
self._install_fn(path, deploy_dir, allow_symlink=True)
else:
with tempfile.TemporaryDirectory() as tmpdir:
# Checkout specified version within local Git repository if this is allowed or clone local Git
# repository to temporary directory and checkout specified version there.
if desc.get('allow use local Git repository'):
tmp_path = path
execute_cmd(self.logger, 'git', '-C', tmp_path, 'checkout', '-fq', version)
execute_cmd(self.logger, 'git', '-C', tmp_path, 'clean', '-xfdq')
else:
tmp_path = os.path.join(tmpdir, os.path.basename(os.path.realpath(path)))
execute_cmd(self.logger, 'git', 'clone', '-q', path, tmp_path)
execute_cmd(self.logger, 'git', '-C', tmp_path, 'checkout', '-q', version)
# Directory .git can be quite large so ignore it during installing except one needs it.
self._install_fn(tmp_path, deploy_dir,
ignore=None if desc.get('copy .git directory') else ['.git'])
elif os.path.isfile(path) and (tarfile.is_tarfile(path) or zipfile.is_zipfile(path)):
os.makedirs(deploy_dir, exist_ok=True)
if tarfile.is_tarfile(path):
self._cmd_fn('tar', '--warning', 'no-unknown-keyword', '-C', '{0}'.format(deploy_dir), '-xf', path)
else:
self._cmd_fn('unzip', '-u', '-d', '{0}'.format(deploy_dir), path)
elif os.path.isfile(path):
self._install_fn(path, instance_path, allow_symlink=True)
elif os.path.isdir(path):
self._install_fn(path, deploy_dir, allow_symlink=True)
else:
self.logger.error(f'Could not install "{name}" since it is provided in the unsupported format')
sys.exit(errno.ENOSYS)
# Remember what entity was installed just if everything went well.
prev_deploy_info[name] = {
'version': version,
'directory': deploy_dir
}
for attr in ('name', 'executable path', 'python path'):
if attr in desc:
prev_deploy_info[name][attr] = desc[attr]
return True
finally:
if tmp_file:
os.unlink(tmp_file)
if tmp_dir:
shutil.rmtree(tmp_dir)
def install(self):
self._pre_install()
execute_cmd(self.logger, 'systemd-tmpfiles', '--create')
install_klever_bridge_production(self.logger, self.args.source_directory, self.args.deployment_directory,
not self._IS_DEV)
self._post_install_or_update(self._IS_DEV)
