def __enter__(self): self.__copy_deployment_files() self.logger.info('Copy sources that can be used during {0}'.format( self.action)) with Cd(self.args.source_directory): try: klever_copy = os.path.join(tempfile.mkdtemp(), 'klever') execute_cmd(self.logger, 'git', 'clone', '.', klever_copy) # Store Klever version to dedicated file and remove directory ".git" since it occupies too much space. with open(os.path.join(klever_copy, 'version'), 'w') as fp: fp.write(setuptools_scm.get_version()) execute_cmd(self.logger, 'rm', '-rf', klever_copy + '/.git') # Development Klever Bridge runs from Klever sources and it creates directories like __pycache__, media, # etc. with root access. We need to backup media and to restore it after update of Klever sources. Other # directories are out of interest, but they should not hinder rsync. sftp = self.ssh.ssh.open_sftp() media_exists = False try: sftp.stat('klever/bridge/media') media_exists = True self.ssh.execute_cmd('mv klever/bridge/media media-backup') self.ssh.execute_cmd('sudo rm -rf klever/bridge') except IOError: pass self.ssh.rsync(klever_copy, '~/') if media_exists: self.ssh.execute_cmd('sudo rm -rf klever/bridge/media') self.ssh.execute_cmd('mv media-backup klever/bridge/media') finally: if os.path.exists(klever_copy): shutil.rmtree(klever_copy)
def _pre_install(self): if os.path.exists(self.prev_deploy_info_file) and not self.keep_addons_and_build_bases: self.logger.error( 'There is information on previous deployment (perhaps you try to install Klever second time)') sys.exit(errno.EINVAL) with open(self.args.deployment_configuration_file) as fp: self.deploy_conf = json.load(fp) self.logger.info('Create deployment directory') os.makedirs(self.args.deployment_directory, exist_ok=True) with open('/etc/default/klever', 'w') as fp: fp.write('KLEVER_SOURCE_DIRECTORY="{0}"\n'.format(os.path.realpath(self.args.source_directory))) fp.write('KLEVER_DEPLOYMENT_DIRECTORY="{0}"\n'.format(os.path.realpath(self.args.deployment_directory))) fp.write('KLEVER_DATA_DIR="{0}"\n' .format(os.path.realpath(self.args.data_directory) if self.args.data_directory else os.path.join(os.path.realpath(self.args.deployment_directory), 'build bases'))) fp.write("KLEVER_WORKERS={}\n".format(os.cpu_count() + 1)) fp.write("KLEVER_PYTHON_BIN_DIR={}\n".format(os.path.dirname(sys.executable))) fp.write("KLEVER_PYTHON={}\n".format(sys.executable)) media_user = get_media_user(self.logger) self.logger.info('Install systemd configuration files and services') execute_cmd(self.logger, 'mkdir', '-p', '/etc/conf.d') for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir, 'systemd', 'conf.d')): for filename in filenames: shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/conf.d', filename)) for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir, 'systemd', 'tmpfiles.d')): for filename in filenames: shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/tmpfiles.d', filename)) replace_media_user(os.path.join('/etc/tmpfiles.d', filename), media_user) for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir, 'systemd', 'system')): for filename in filenames: shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/systemd/system', filename)) replace_media_user(os.path.join('/etc/systemd/system', filename), media_user) self._install_or_update_deps() prepare_env(self.logger, self.args.deployment_directory) self._pre_install_or_update() # Set environment variable JAVA to point out absolute path to java executable to be used for executing Java # programs within Klever. At the moment the same java will be used for all Java programs but that may be changed # in future. with open('/etc/default/klever', 'a+') as fp: fp.write("JAVA={}\n".format( os.path.join(os.path.realpath(self.args.deployment_directory), 'klever-addons', 'JRE', self.prev_deploy_info['Klever Addons']['JRE']['executable path'], 'java')))
def open_shell(self): self.logger.info( 'Open interactive SSH to instance "{0}" (IP: {1})'.format( self.name, self.floating_ip)) execute_cmd(self.logger, 'ssh', '-o', 'StrictHostKeyChecking=no', '-i', self.args.ssh_rsa_private_key_file, '{0}@{1}'.format(self.args.ssh_username, self.floating_ip), keep_stdout=True)
def __get_build_base_version(self, klever_build_base): version = self.deploy_conf['Klever Build Bases'][klever_build_base].get('version') if version: return version klever_build_base_path = self.deploy_conf['Klever Build Bases'][klever_build_base]['path'] if os.path.isfile(klever_build_base_path): # Use md5 checksum of the archive as version output = execute_cmd(self.logger, 'md5sum', klever_build_base_path, stderr=subprocess.DEVNULL, get_output=True).rstrip() version = output.split(' ')[0] elif os.path.isdir(klever_build_base_path): # Use unique identifier of the build base as version try: version = Clade(klever_build_base_path).get_uuid() except RuntimeError: self.logger.error(f'"{klever_build_base}" is not a valid Clade build base') sys.exit(errno.EINVAL) else: # Otherwise build base is probably a link to the remote file # Our build bases are mostly stored at redmine, which has unique links # Here we use this link as version version = klever_build_base_path return version
def __enter__(self): self.__copy_deployment_files() self.logger.info('Copy sources that can be used during {0}'.format(self.action)) with Cd(self.args.source_directory): try: klever_copy = os.path.join(tempfile.mkdtemp(), 'klever') execute_cmd(self.logger, 'git', 'clone', '.', klever_copy) # Store Klever version to dedicated file and remove directory ".git" since it occupies too much space. with open(os.path.join(klever_copy, 'version'), 'w') as fp: fp.write(setuptools_scm.get_version()) execute_cmd(self.logger, 'rm', '-rf', klever_copy + '/.git') self.ssh.rsync(klever_copy, '~/') finally: if os.path.exists(klever_copy): shutil.rmtree(klever_copy)
def _install_fn(self, src, dst, allow_symlink=False, ignore=None): if ignore and allow_symlink: self.logger.error('You can not both use symbolic links and ignore some directories') sys.exit(errno.EINVAL) self.logger.info('Install "{0}" to "{1}"'.format(src, dst)) os.makedirs(dst if os.path.isdir(dst) else os.path.dirname(dst), exist_ok=True) if allow_symlink and self.args.allow_symbolic_links: execute_cmd(self.logger, 'ln', '-s', src, dst) else: if os.path.isdir(src): shutil.copytree(src, dst, symlinks=True, ignore=lambda source, names: ignore or []) else: shutil.copy(src, dst)
def rsync(self, host_path, instance_path): if instance_path: self.execute_cmd(f'mkdir -p {instance_path}') else: instance_path = "~/" # stderr=subprocess.DEVNULL is required to suppress WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! # maybe there is a better way to fix it execute_cmd( self.logger, 'rsync', '-ar', '-e', f'ssh -o StrictHostKeyChecking=no -i {self.args.ssh_rsa_private_key_file}', host_path, f'{OS_USER}@{self.floating_ip}:{instance_path}', stderr=subprocess.DEVNULL )
def _pre_install(self): if self.prev_deploy_info: self.logger.error( 'There is information on previous deployment (perhaps you try to install Klever second time)') sys.exit(errno.EINVAL) with open(self.args.deployment_configuration_file) as fp: self.deploy_conf = json.load(fp) self.logger.info('Create deployment directory') os.makedirs(self.args.deployment_directory, exist_ok=True) with open('/etc/default/klever', 'w') as fp: fp.write('KLEVER_SOURCE_DIRECTORY="{0}"\n'.format(os.path.realpath(self.args.source_directory))) fp.write('KLEVER_DEPLOYMENT_DIRECTORY="{0}"\n'.format(os.path.realpath(self.args.deployment_directory))) fp.write('KLEVER_DATA_DIR="{0}"\n' .format(os.path.join(os.path.realpath(self.args.deployment_directory), 'klever', 'build bases') if len(self.deploy_conf['Klever Build Bases']) else os.path.join(os.path.realpath(self.args.source_directory), 'build bases'))) fp.write("KLEVER_WORKERS={}\n".format(os.cpu_count() + 1)) fp.write("KLEVER_PYTHON_BIN_DIR={}\n".format(os.path.dirname(sys.executable))) fp.write("KLEVER_PYTHON={}\n".format(sys.executable)) media_user = get_media_user(self.logger) self.logger.info('Install systemd configuration files and services') execute_cmd(self.logger, 'mkdir', '-p', '/etc/conf.d') for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir, 'systemd', 'conf.d')): for filename in filenames: shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/conf.d', filename)) for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir, 'systemd', 'tmpfiles.d')): for filename in filenames: shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/tmpfiles.d', filename)) replace_media_user(os.path.join('/etc/tmpfiles.d', filename), media_user) for dirpath, _, filenames in os.walk(os.path.join(os.path.dirname(__file__), os.path.pardir, 'systemd', 'system')): for filename in filenames: shutil.copy(os.path.join(dirpath, filename), os.path.join('/etc/systemd/system', filename)) replace_media_user(os.path.join('/etc/systemd/system', filename), media_user) self._install_or_update_deps() prepare_env(self.logger, self.args.deployment_directory) self._pre_install_or_update()
def __enter__(self): self.logger.info('Copy deployment configuration file') self.ssh.sftp_put(self.args.deployment_configuration_file, 'klever.json') self.logger.info('Copy sources that can be used during {0}'.format(self.action)) with Cd(self.args.source_directory): try: execute_cmd(self.logger, 'git', 'clone', '.', '__klever') execute_cmd(self.logger, 'tar', '-C', '__klever', '-cf', '__klever.tar.gz', '.') self.ssh.sftp_put('__klever.tar.gz', 'klever/klever.tar.gz') self.ssh.execute_cmd('tar --warning no-unknown-keyword -C klever -xf klever/klever.tar.gz') self.ssh.execute_cmd('rm klever/klever.tar.gz') finally: if os.path.exists('__klever'): shutil.rmtree('__klever') if os.path.exists('__klever.tar.gz'): os.remove('__klever.tar.gz')
def rsync(self, host_path, instance_path): if not instance_path: instance_path = "~/" if instance_path.startswith('~'): # with '-s' rsync sends all filenames without allowing the remote shell to interpret them # so, we need to explicitly expand ~ here instance_path = instance_path.replace('~', f'/home/{OS_USER}', 1) self.logger.debug( 'Execute rsync command to instance "{}" (IP: {})\ncopy {} to {}'. format(self.name, self.floating_ip, host_path, instance_path)) # mkdir also doesn't work with paths inside quotes that contain ~ self.execute_cmd(f'mkdir -p "{instance_path}"') if os.path.isfile(host_path) and (tarfile.is_tarfile(host_path) or zipfile.is_zipfile(host_path)): rsync_flags = '-as' else: # with '-z' rsync compresses the transmitted data rsync_flags = '-asz' # stderr=subprocess.DEVNULL is required to suppress WARNING: REMOTE HOST IDENTIFICATION HAS CHANGED! # maybe there is a better way to fix it execute_cmd( self.logger, 'rsync', rsync_flags, '--del', '-e', f'ssh -o StrictHostKeyChecking=no -i {self.args.ssh_rsa_private_key_file}', # Exclude Python build directories since rsync can fail to remove them. Indeed, this is a workaround that is # necessary due to we incorrectly install Klever on OpenStack instances using the superuser rather than the # virtual environment. '--exclude', 'build', '--exclude', 'klever.egg-info', host_path, f'{OS_USER}@{self.floating_ip}:{instance_path}', stderr=subprocess.DEVNULL)
def __enter__(self): self.logger.info('Copy deployment configuration file') self.ssh.sftp_put(self.args.deployment_configuration_file, 'klever.json') self.logger.info('Copy sources that can be used during {0}'.format( self.action)) with Cd(self.args.source_directory): try: execute_cmd(self.logger, 'git', 'clone', '.', '__klever') # Store Klever version to dedicated file and remove directory ".git" since it occupies too much space. with Cd('__klever'): version = get_version() with open('version', 'w') as fp: fp.write(version) execute_cmd(self.logger, 'rm', '-rf', '__klever/.git') execute_cmd(self.logger, 'tar', '-C', '__klever', '-cf', '__klever.tar.gz', '.') self.ssh.sftp_put('__klever.tar.gz', 'klever/klever.tar.gz') self.ssh.execute_cmd( 'tar --warning no-unknown-keyword -C klever -xf klever/klever.tar.gz' ) self.ssh.execute_cmd('rm klever/klever.tar.gz') finally: if os.path.exists('__klever'): shutil.rmtree('__klever') if os.path.exists('__klever.tar.gz'): os.remove('__klever.tar.gz')
def prepare_env(logger, deploy_dir): logger.info('Prepare environment') try: logger.debug('Try to create user "klever"') execute_cmd(logger, 'useradd', 'klever') except subprocess.CalledProcessError: logger.debug('User "klever" already exists') try: logger.debug('Obtain execute access to {!r} home directory'.format(os.getlogin())) execute_cmd(logger, 'chmod', 'o+x', os.path.join('/', 'home', os.getlogin())) except Exception: pass logger.debug('Prepare configurations directory') execute_cmd(logger, 'mkdir', os.path.join(deploy_dir, 'klever-conf')) logger.debug('Prepare working directory') work_dir = os.path.join(deploy_dir, 'klever-work') execute_cmd(logger, 'mkdir', work_dir) execute_cmd(logger, 'chown', '-LR', 'klever', work_dir) openssl_header = '/usr/include/openssl/opensslconf.h' if not os.path.exists(openssl_header): logger.debug('Create soft links for libssl to build new versions of the Linux kernel') execute_cmd(logger, 'ln', '-s', '/usr/include/x86_64-linux-gnu/openssl/opensslconf.h', openssl_header) crts = glob.glob('/usr/lib/x86_64-linux-gnu/crt*.o') args = [] for crt in crts: if not os.path.exists(os.path.join('/usr/lib', os.path.basename(crt))): args.append(crt) if args: logger.debug('Prepare CIF environment') args.append('/usr/lib') execute_cmd(logger, 'ln', '-s', *args) logger.debug('Try to initialise PostgreSQL') try: execute_cmd(logger, 'postgresql-setup', '--initdb', '--unit', 'postgresql') except FileNotFoundError: # postgresql-setup may not be present in the system. On some systems like openSUSE it is necessary to start the # PostgreSQL service at least once so that necessary initialization will be performed automatically. execute_cmd(logger, 'service', 'postgresql', 'restart') except subprocess.CalledProcessError: # postgresql-setup may fail if it was already executed before pass # Search for pg_hba.conf in all possible locations pg_hba_conf_file = None for path in ('/etc/postgresql', '/var/lib/pgsql/data'): try: pg_hba_conf_file = execute_cmd(logger, 'find', path, '-name', 'pg_hba.conf', get_output=True).rstrip() except subprocess.CalledProcessError: continue with open(pg_hba_conf_file) as fp: pg_hba_conf = fp.readlines() with open(pg_hba_conf_file, 'w') as fp: for line in pg_hba_conf: # change ident to md5 if line.split() == ['host', 'all', 'all', '127.0.0.1/32', 'ident']: line = 'host all all 127.0.0.1/32 md5\n' fp.write(line) execute_cmd(logger, 'service', 'postgresql', 'restart') if not pg_hba_conf_file: logger.error('Could not find PostgreSQL configuration file') sys.exit(errno.EINVAL) logger.debug('Start and enable PostgreSQL service') execute_cmd(logger, 'systemctl', 'start', 'postgresql') execute_cmd(logger, 'systemctl', 'enable', 'postgresql') logger.debug('Create PostgreSQL user') execute_cmd(logger, 'psql', '-c', "CREATE USER klever WITH CREATEDB PASSWORD 'klever'", username='******') logger.debug('Create PostgreSQL database') execute_cmd(logger, 'createdb', '-T', 'template0', '-E', 'utf-8', 'klever', username='******') logger.debug('Start and enable RabbitMQ server service') execute_cmd(logger, 'systemctl', 'start', 'rabbitmq-server.service') execute_cmd(logger, 'systemctl', 'enable', 'rabbitmq-server.service') logger.debug('Create RabbitMQ user') execute_cmd(logger, 'rabbitmqctl', 'add_user', 'service', 'service') execute_cmd(logger, 'rabbitmqctl', 'set_user_tags', 'service', 'administrator') execute_cmd(logger, 'rabbitmqctl', 'set_permissions', '-p', '/', 'service', '.*', '.*', '.*')
def install_klever_bridge_production(logger, src_dir, deploy_dir, populate_just_production_presets=True, update=False): logger.info('Install/update production Klever Bridge') services = ('nginx', 'klever-bridge', 'klever-celery', 'klever-celerybeat') stop_services(logger, services) logger.info('Copy Klever Bridge configuration file for NGINX') copy_from = os.path.join(src_dir, 'bridge/conf/nginx') if os.path.exists('/etc/nginx/sites-enabled'): shutil.copy(copy_from, '/etc/nginx/sites-enabled/klever-bridge.conf') else: shutil.copy(copy_from, '/etc/nginx/conf.d/klever-bridge.conf') logger.info('Install/update Klever Bridge source/binary code') shutil.rmtree('/var/www/klever-bridge', ignore_errors=True) shutil.copytree(os.path.join(src_dir, 'bridge'), '/var/www/klever-bridge/bridge', ignore=shutil.ignore_patterns('test_files')) shutil.copytree(os.path.join(src_dir, 'presets'), '/var/www/klever-bridge/presets') logger.info('Prepare media directory') media = '/var/www/klever-bridge/bridge/media' media_real = os.path.realpath( os.path.join(os.path.realpath(deploy_dir), 'klever-media')) shutil.rmtree(media) execute_cmd(logger, 'mkdir', '-p', media_real) execute_cmd(logger, 'ln', '-s', '-T', media_real, media) with Cd('/var/www/klever-bridge/bridge'): with open('bridge/settings.py', 'w') as fp: fp.write('from bridge.{0} import *\n'.format('production')) if not populate_just_production_presets: fp.write('POPULATE_JUST_PRODUCTION_PRESETS = False\n') _install_klever_bridge(logger, update) logger.info('Collect static files') execute_cmd(logger, sys.executable, './manage.py', 'collectstatic', '--noinput') # Make available data from media, logs and static for its actual user. media_user = get_media_user(logger) user_group = '{}:{}'.format(media_user, media_user) execute_cmd(logger, 'chown', '-R', user_group, media_real) execute_cmd(logger, 'chown', '-R', user_group, '/var/www/klever-bridge/bridge/logs') execute_cmd(logger, 'chown', '-R', user_group, '/var/www/klever-bridge/bridge/static') # Try to add httpd_t to the list of permissive domains. try: execute_cmd(logger, 'semanage', 'permissive', '-a', 'httpd_t') except Exception: pass start_services(logger, services)
def _install_klever_bridge(logger, update): logger.info('Update translations') execute_cmd(logger, sys.executable, './manage.py', 'compilemessages') logger.info('Migrate database') execute_cmd(logger, sys.executable, './manage.py', 'migrate') logger.info('Populate database') # We need to create users once. Otherwise this can overwrite their settings changed manually. if not update: execute_cmd(logger, sys.executable, './manage.py', 'createuser', '--username', 'admin', '--password', 'admin', '--staff', '--superuser') execute_cmd(logger, sys.executable, './manage.py', 'createuser', '--username', 'manager', '--password', 'manager', '--role', '2') execute_cmd(logger, sys.executable, './manage.py', 'createuser', '--username', 'service', '--password', 'service', '--role', '4') execute_cmd(logger, sys.executable, './manage.py', 'populate', '--all')
def install_deps(logger, deploy_conf, prev_deploy_info, non_interactive, update_pckgs): if non_interactive: # Do not require users input. os.environ['DEBIAN_FRONTEND'] = 'noninteractive' # Get packages to be installed/updated. pckgs_to_install = [] pckgs_to_update = [] deploy_conf['Packages'] = load_deps_conf(logger).strip().split(' ') if 'Packages' in prev_deploy_info: for pckg in deploy_conf['Packages']: if pckg in prev_deploy_info['Packages']: pckgs_to_update.append(pckg) else: pckgs_to_install.append(pckg) else: # All packages should be installed. pckgs_to_install = deploy_conf['Packages'] if pckgs_to_install or (pckgs_to_update and update_pckgs): logger.info('Update packages list') if shutil.which('apt'): execute_cmd(logger, 'apt', 'update') elif shutil.which('dnf'): execute_cmd(logger, 'dnf', 'update') elif shutil.which('zypper'): execute_cmd(logger, 'zypper', 'ref') elif shutil.which('yum'): execute_cmd(logger, 'yum', 'check-update') else: logger.error('Your Linux distribution is not supported') sys.exit(errno.EINVAL) if pckgs_to_install: logger.info('Install packages:\n {0}'.format( '\n '.join(pckgs_to_install))) for util in ('apt', 'dnf', 'zypper', 'yum'): if shutil.which(util): args = [util, 'install'] if non_interactive: args.append('-y') args.extend(pckgs_to_install) execute_cmd(logger, *args) break else: logger.error('Your Linux distribution is not supported') sys.exit(errno.EINVAL) # Remember what packages were installed just if everything went well. if 'Packages' not in prev_deploy_info: prev_deploy_info['Packages'] = [] prev_deploy_info['Packages'] = sorted(prev_deploy_info['Packages'] + pckgs_to_install) if pckgs_to_update and update_pckgs: logger.info('Update packages:\n {0}'.format( '\n '.join(pckgs_to_update))) for util in ('apt', 'dnf', 'zypper', 'yum'): if shutil.which(util): if util in ('apt', 'dnf'): args = [util, 'upgrade'] elif util in ('yum', 'zypper'): args = [util, 'update'] if non_interactive: args.append('-y') args.extend(pckgs_to_install) execute_cmd(logger, *args) break else: raise RuntimeError('Your Linux distribution is not supported')
def _cmd_fn(self, *args): execute_cmd(self.logger, *args)
def install(self): self._pre_install() execute_cmd(self.logger, 'systemd-tmpfiles', '--create') self._install_or_update() self._post_install_or_update(self._IS_DEV)
def _pre_uninstall(self, mode_services): services = list(mode_services) services.extend( ('klever-controller', 'klever-native-scheduler', 'klever-cgroup')) if need_verifiercloud_scheduler(self.prev_deploy_info): services.append('klever-verifiercloud-scheduler') stop_services(self.logger, services, ignore_errors=True) self.logger.info('Uninstall systemd services') for dirpath, _, filenames in os.walk('/etc/systemd/system'): for filename in filenames: if filename.startswith('klever'): service = os.path.join(dirpath, filename) self.logger.info('Remove "{0}"'.format(service)) os.remove(service) klever_env_file = '/etc/default/klever' if os.path.exists(klever_env_file): self.logger.info('Remove "{0}"'.format(klever_env_file)) os.remove(klever_env_file) # Remove bridge files bridge_path = os.path.join(self.args.deployment_directory, 'klever/bridge/bridge') for path in ('settings.py', 'db.json', 'rmq.json'): path = os.path.join(bridge_path, path) if os.path.exists(path): self.logger.info('Remove "{0}"'.format(path)) os.remove(path) # Removing individual directories and files rather than the whole deployment directory allows to use standard # locations like "/", "/usr" or "/usr/local" for deploying Klever. for path in ('klever', 'klever-addons', 'klever-conf', 'klever-work', 'klever-media', 'klever.json'): path = os.path.join(self.args.deployment_directory, path) if os.path.exists(path) or os.path.islink(path): self.logger.info('Remove "{0}"'.format(path)) if os.path.islink(path) or os.path.isfile(path): os.remove(path) else: shutil.rmtree(path) # Remove deployment directory if it is empty if os.path.exists(self.args.deployment_directory) and not os.listdir( self.args.deployment_directory): self.logger.info('Remove "{0}"'.format( self.args.deployment_directory)) os.rmdir(self.args.deployment_directory) # Remove Klever Bridge NGINX configuration if so. for klever_bridge_nginx_conf_file in ( '/etc/nginx/sites-enabled/klever-bridge.conf', '/etc/nginx/conf.d/klever-bridge.conf'): if os.path.exists(klever_bridge_nginx_conf_file): self.logger.info( 'Remove "{0}"'.format(klever_bridge_nginx_conf_file)) os.remove(klever_bridge_nginx_conf_file) stop_services(self.logger, ('nginx', )) start_services(self.logger, ('nginx', )) try: pwd.getpwnam('postgres') except KeyError: # Do nothing if user "postgres" does not exist. pass else: self.logger.info('Delete PostgreSQL database') execute_cmd(self.logger, 'dropdb', '--if-exists', 'klever', username='******') self.logger.info('Delete PostgreSQL user') execute_cmd(self.logger, 'psql', '-c', "DROP USER IF EXISTS klever", username='******') try: pwd.getpwnam('klever') except KeyError: # Do nothing if user "klever" does not exist. pass else: self.logger.info('Delete user "klever"') execute_cmd(self.logger, 'userdel', 'klever') try: self.logger.info('Delete RabbitMQ user') execute_cmd(self.logger, 'rabbitmqctl', 'delete_user', 'service') except (FileNotFoundError, subprocess.CalledProcessError): pass # Try to remove httpd_t from the list of permissive domains. try: execute_cmd(self.logger, 'semanage', 'permissive', '-d', 'httpd_t') except Exception: pass
def _install_klever_bridge(logger): logger.info('Update translations') execute_cmd(logger, sys.executable, './manage.py', 'compilemessages') logger.info('Migrate database') execute_cmd(logger, sys.executable, './manage.py', 'migrate') logger.info('Populate database') execute_cmd(logger, sys.executable, './manage.py', 'createuser', '--username', 'admin', '--password', 'admin', '--staff', '--superuser') execute_cmd(logger, sys.executable, './manage.py', 'createuser', '--username', 'manager', '--password', 'manager', '--role', '2') execute_cmd(logger, sys.executable, './manage.py', 'createuser', '--username', 'service', '--password', 'service', '--role', '4') execute_cmd(logger, sys.executable, './manage.py', 'populate', '--all')
def _install_entity(self, name, src_dir, deploy_dir, deploy_conf, prev_deploy_info, build_base=False): if name not in deploy_conf: self.logger.error(f'"{name}" is not described') sys.exit(errno.EINVAL) deploy_dir = os.path.normpath(deploy_dir) desc = deploy_conf[name] if 'version' not in desc: self.logger.error(f'Version is not specified for "{name}"') sys.exit(errno.EINVAL) version = desc['version'] if 'path' not in desc: self.logger.error(f'Path is not specified for "{name}"') sys.exit(errno.EINVAL) path = desc['path'] o = urllib.parse.urlparse(path) if not o[0]: path = make_canonical_path(src_dir, path) refs = {} try: ref_strs = execute_cmd(self.logger, 'git', 'ls-remote', '--refs', path, stderr=subprocess.DEVNULL, get_output=True).rstrip().split('\n') is_git_repo = True for ref_str in ref_strs: commit, ref = ref_str.split('\t') refs[ref] = commit except subprocess.CalledProcessError: is_git_repo = False if is_git_repo and version != 'CURRENT': # Version can be either some reference or commit hash. In the former case we need to get corresponding # commit hash since it can differ from the previous one installed before for the same reference. In the # latter case we will fail below one day if commit hash isn't valid. # Note that here we can use just Git commands working with remote repositories since we didn't clone them # yet and we don't want do this if update isn't necessary. for prefix in ('refs/heads/', 'refs/tags/'): if prefix + version in refs: version = refs[prefix + version] break prev_version = prev_deploy_info[name]['version'] if name in prev_deploy_info else None if version == prev_version and version != 'CURRENT': self.logger.info(f'"{name}" is up to date (version: "{version}")') return False entity_kind = "Klever build base" if build_base else "Klever addon" if prev_version: self.logger.info(f'Update {entity_kind} "{name}" from version "{prev_version}" to version "{version}"') else: self.logger.info(f'Install {entity_kind} "{name}" (version: "{version}")') # Remove previous version of entity if so. Do not make this in depend on previous version since it can be unset # while entity is deployed. For instance, this can be the case when entity deployment fails somewhere in the # middle. self._cmd_fn('rm', '-rf', deploy_dir) # Install new version of entity. tmp_file = None tmp_dir = None try: instance_path = os.path.join(deploy_dir, os.path.basename(path)) # Clone remote Git repository. if (o[0] == 'git' or is_git_repo) and not os.path.exists(path): tmp_dir = tempfile.mkdtemp() execute_cmd(self.logger, 'git', 'clone', '-q', '--recursive', path, tmp_dir) path = tmp_dir # Download remote file. elif o[0] in ('http', 'https', 'ftp'): _, tmp_file = tempfile.mkstemp() execute_cmd(self.logger, 'wget', '-O', tmp_file, '-q', path) path = tmp_file elif o[0]: self.logger.error(f'"{name}" is provided in unsupported form "{o[0]}"') sys.exit(errno.EINVAL) elif not os.path.exists(path): self.logger.error(f'Path "{path}" does not exist') sys.exit(errno.ENOENT) if is_git_repo: if version == 'CURRENT': self._install_fn(path, deploy_dir, allow_symlink=True) else: with tempfile.TemporaryDirectory() as tmpdir: # Checkout specified version within local Git repository if this is allowed or clone local Git # repository to temporary directory and checkout specified version there. if desc.get('allow use local Git repository'): tmp_path = path execute_cmd(self.logger, 'git', '-C', tmp_path, 'checkout', '-fq', version) execute_cmd(self.logger, 'git', '-C', tmp_path, 'clean', '-xfdq') else: tmp_path = os.path.join(tmpdir, os.path.basename(os.path.realpath(path))) execute_cmd(self.logger, 'git', 'clone', '-q', path, tmp_path) execute_cmd(self.logger, 'git', '-C', tmp_path, 'checkout', '-q', version) # Directory .git can be quite large so ignore it during installing except one needs it. self._install_fn(tmp_path, deploy_dir, ignore=None if desc.get('copy .git directory') else ['.git']) elif os.path.isfile(path) and (tarfile.is_tarfile(path) or zipfile.is_zipfile(path)): os.makedirs(deploy_dir, exist_ok=True) if tarfile.is_tarfile(path): self._cmd_fn('tar', '--warning', 'no-unknown-keyword', '-C', '{0}'.format(deploy_dir), '-xf', path) else: self._cmd_fn('unzip', '-u', '-d', '{0}'.format(deploy_dir), path) elif os.path.isfile(path): self._install_fn(path, instance_path, allow_symlink=True) elif os.path.isdir(path): self._install_fn(path, deploy_dir, allow_symlink=True) else: self.logger.error(f'Could not install "{name}" since it is provided in the unsupported format') sys.exit(errno.ENOSYS) # Remember what entity was installed just if everything went well. prev_deploy_info[name] = { 'version': version, 'directory': deploy_dir } for attr in ('name', 'executable path', 'python path'): if attr in desc: prev_deploy_info[name][attr] = desc[attr] return True finally: if tmp_file: os.unlink(tmp_file) if tmp_dir: shutil.rmtree(tmp_dir)
def install(self): self._pre_install() execute_cmd(self.logger, 'systemd-tmpfiles', '--create') install_klever_bridge_production(self.logger, self.args.source_directory, self.args.deployment_directory, not self._IS_DEV) self._post_install_or_update(self._IS_DEV)