def post_update(cls, cron): config = Config() # When `cron` is True, we want to bypass question and just recreate # YML and environment files from new templates if cron is True: current_dict = config.get_template() current_dict.update(config.get_dict()) config.set_config(current_dict) Template.render(config, force=True) sys.exit(0) message = ('After an update, it is strongly recommended to run\n' '`python3 run.py --setup` to regenerate environment files.') CLI.framed_print(message, color=CLI.COLOR_INFO) response = CLI.yes_no_question('Do you want to proceed?') if response is True: current_dict = config.build() Template.render(config) config.init_letsencrypt() Setup.update_hosts(current_dict) question = 'Do you want to (re)start containers?' response = CLI.yes_no_question(question) if response is True: Command.start()
def read_unique_id(self): """ Reads unique id from file `Config.UNIQUE_ID_FILE` Returns: str """ unique_id = None try: unique_id_file = os.path.join(self.__dict['support_api_path'], Config.UNIQUE_ID_FILE) except KeyError: if self.first_time: return None else: CLI.framed_print( 'Bad configuration! The path of support_api ' 'path is missing. Please delete `.run.conf` ' 'and start from scratch', color=CLI.COLOR_ERROR) sys.exit(1) try: with open(unique_id_file, 'r') as f: unique_id = f.read().strip() except FileNotFoundError: pass return unique_id
def run(force_setup=False): if sys.version_info[0] == 2: message = ( 'DEPRECATION: Python 2.7 has reached the end of its life on ' 'January 1st, 2020. Please upgrade your Python as Python 2.7 is ' 'not maintained anymore.\n\n' 'A future version of KoBoInstall will drop support for it.') CLI.framed_print(message) if not platform.system() in ['Linux', 'Darwin']: CLI.colored_print('Not compatible with this OS', CLI.COLOR_ERROR) else: config = Config() dict_ = config.get_dict() if config.first_time: force_setup = True if force_setup: dict_ = config.build() Setup.clone_kobodocker(config) Template.render(config) config.init_letsencrypt() Setup.update_hosts(dict_) else: if config.auto_detect_network(): Template.render(config) Setup.update_hosts(dict_) Command.start()
def render(cls, config, force=False): """ Write configuration files based on `config` Args: config (helpers.config.Config) force (bool) """ dict_ = config.get_dict() template_variables = cls.__get_template_variables(config) environment_directory = config.get_env_files_path() unique_id = cls.__read_unique_id(environment_directory) if (not force and unique_id and str(dict_.get('unique_id', '')) != str(unique_id)): message = ( 'WARNING!\n\n' 'Existing environment files are detected. Files will be ' 'overwritten.') CLI.framed_print(message) response = CLI.yes_no_question('Do you want to continue?', default=False) if not response: sys.exit(0) cls.__write_unique_id(environment_directory, dict_['unique_id']) base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) templates_path_parent = os.path.join(base_dir, 'templates') # Environment templates_path = os.path.join(templates_path_parent, Config.ENV_FILES_DIR, '') for root, dirnames, filenames in os.walk(templates_path): destination_directory = cls.__create_directory( environment_directory, root, templates_path) cls.__write_templates(template_variables, root, destination_directory, filenames) # kobo-docker templates_path = os.path.join(templates_path_parent, 'kobo-docker') for root, dirnames, filenames in os.walk(templates_path): destination_directory = dict_['kobodocker_path'] cls.__write_templates(template_variables, root, destination_directory, filenames) # nginx-certbox if config.use_letsencrypt: templates_path = os.path.join(templates_path_parent, Config.LETSENCRYPT_DOCKER_DIR, '') for root, dirnames, filenames in os.walk(templates_path): destination_directory = cls.__create_directory( config.get_letsencrypt_repo_path(), root, templates_path) cls.__write_templates(template_variables, root, destination_directory, filenames)
def __welcome(): message = ( 'Welcome to SUPPORT API for KoBoToolbox.\n' '\n' 'You are going to be asked some questions that will determine how ' 'to build the configuration of `Support API`.\n' '\n' 'Some questions already have default values (within brackets).\n' 'Just press `enter` to accept the default value or enter `-` to ' 'remove previously entered value.\n' 'Otherwise choose between choices or type your answer. ') CLI.framed_print(message, color=CLI.COLOR_INFO)
def __validate_custom_yml(config, command): """ Validate whether docker-compose must start the containers with a custom YML file in addition to the default. If the file does not yet exist, kobo-install is paused until the user creates it and resumes the setup manually. If user has chosen to use a custom YML file, it is injected into `command` before being executed. """ dict_ = config.get_dict() frontend_command = True # Detect if it's a front-end command or back-end command for part in command: if 'backend' in part: frontend_command = False break if frontend_command and dict_['use_frontend_custom_yml']: custom_file = '{}/docker-compose.frontend.custom.yml'.format( dict_['kobodocker_path']) does_custom_file_exist = os.path.exists(custom_file) while not does_custom_file_exist: message = ('Please create your custom configuration in\n' '`{custom_file}`.').format(custom_file=custom_file) CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90) input('Press any key when it is done...') does_custom_file_exist = os.path.exists(custom_file) # Add custom file to docker-compose command command.insert(5, '-f') command.insert(6, 'docker-compose.frontend.custom.yml') if not frontend_command and dict_['use_backend_custom_yml']: backend_server_role = dict_['backend_server_role'] custom_file = '{}/docker-compose.backend.{}.custom.yml'.format( dict_['kobodocker_path'], backend_server_role) does_custom_file_exist = os.path.exists(custom_file) while not does_custom_file_exist: message = ('Please create your custom configuration in\n' '`{custom_file}`.').format(custom_file=custom_file) CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90) input('Press any key when it is done...') does_custom_file_exist = os.path.exists(custom_file) # Add custom file to docker-compose command command.insert(5, '-f') command.insert( 6, 'docker-compose.backend.{}.custom.yml'.format( backend_server_role))
def __validate_installation(self): """ Validates if installation is not run over existing data. The check is made only the first time the setup is run. :return: bool """ if self.first_time: postgres_dir_path = os.path.join(self.__dict['support_api_path'], '.vols', 'db') postgres_data_exists = os.path.exists( postgres_dir_path) and os.path.isdir(postgres_dir_path) if postgres_data_exists: # Not a reliable way to detect whether folder contains # kobo-install files. We assume that if # `docker-compose.backend.template.yml` is there, Docker # images are the good ones. # TODO Find a better way docker_composer_file_path = os.path.join( self.__dict['support_api_path'], 'docker-compose.backend.template.yml') if not os.path.exists(docker_composer_file_path): message = ( 'WARNING!\n\n' 'You are installing over existing data.\n' '\n' 'It is recommended to backup your data and import it ' 'to a fresh installed (by Support API install) database.\n' '\n' 'support-api-install uses these images:\n' ' - PostgreSQL: mdillon/postgis:9.5\n' '\n' 'Be sure to upgrade to these versions before going ' 'further!') CLI.framed_print(message) response = CLI.yes_no_question( 'Are you sure you want to continue?', default=False) if response is False: sys.exit(0) else: CLI.colored_print( 'Privileges escalation is needed to prepare DB', CLI.COLOR_WARNING) # Write `kobo_first_run` file to run postgres # container's entrypoint flawlessly. os.system( 'echo $(date) | sudo tee -a {} > /dev/null'.format( os.path.join(self.__dict['support_api_path'], '.vols', 'db', 'kobo_first_run')))
def display_error_message(message): message += '\nPlease run `python3 run.py --setup` first.' CLI.framed_print(message, color=CLI.COLOR_ERROR) sys.exit(1)
def update_hosts(cls, dict_): """ Args: dict_ (dict): Dictionary provided by `Config.get_dict()` """ if dict_['local_installation']: start_sentence = '### (BEGIN) KoBoToolbox local routes' end_sentence = '### (END) KoBoToolbox local routes' with open('/etc/hosts', 'r') as f: tmp_host = f.read() start_position = tmp_host.find(start_sentence) end_position = tmp_host.find(end_sentence) if start_position > -1: tmp_host = tmp_host[0: start_position] \ + tmp_host[end_position + len(end_sentence) + 1:] routes = '{ip_address} ' \ '{kpi_subdomain}.{public_domain_name} ' \ '{kc_subdomain}.{public_domain_name} ' \ '{ee_subdomain}.{public_domain_name}'.format( ip_address=dict_['local_interface_ip'], public_domain_name=dict_['public_domain_name'], kpi_subdomain=dict_['kpi_subdomain'], kc_subdomain=dict_['kc_subdomain'], ee_subdomain=dict_['ee_subdomain'] ) tmp_host = ('{bof}' '\n{start_sentence}' '\n{routes}' '\n{end_sentence}').format( bof=tmp_host.strip(), start_sentence=start_sentence, routes=routes, end_sentence=end_sentence) with open('/tmp/etchosts', 'w') as f: f.write(tmp_host) message = ('Privileges escalation is required to update ' 'your `/etc/hosts`.') CLI.framed_print(message, color=CLI.COLOR_INFO) dict_['review_host'] = CLI.yes_no_question( 'Do you want to review your /etc/hosts file ' 'before overwriting it?', default=dict_['review_host']) if dict_['review_host']: print(tmp_host) CLI.colored_input('Press any keys when ready') # Save 'review_host' config = Config() config.write_config() cmd = 'sudo mv /etc/hosts /etc/hosts.old ' \ '&& sudo mv /tmp/etchosts /etc/hosts' return_value = os.system(cmd) if return_value != 0: sys.exit(1)
#!/usr/bin/env python3 # -*- coding: utf-8 -*- import platform import sys from helpers.cli import CLI if sys.version_info[0] == 2: message = ( 'Python 2.7 has reached the end of its life on ' 'January 1st, 2020. Please upgrade your Python as Python 2.7 is ' 'not maintained anymore.') CLI.framed_print(message, color=CLI.COLOR_ERROR) sys.exit(1) from helpers.command import Command from helpers.config import Config from helpers.setup import Setup from helpers.template import Template from helpers.updater import Updater from helpers.upgrading import Upgrading def run(force_setup=False): if not platform.system() in ['Linux', 'Darwin']: CLI.colored_print('Not compatible with this OS', CLI.COLOR_ERROR) else: config = Config() dict_ = config.get_dict() if config.first_time: force_setup = True
def migrate_single_to_two_databases(config): """ Check the contents of the databases. If KPI's is empty or doesn't exist while KoBoCAT's has user data, then we are migrating from a single-database setup Args config (helpers.config.Config) """ dict_ = config.get_dict() backend_role = dict_['backend_server_role'] def _kpi_db_alias_kludge(command): """ Sorry, this is not very nice. See https://github.com/kobotoolbox/kobo-docker/issues/264. """ set_env = 'DATABASE_URL="${KPI_DATABASE_URL}"' return ['bash', '-c', '{} {}'.format(set_env, command)] kpi_run_command = [ 'docker-compose', '-f', 'docker-compose.frontend.yml', '-f', 'docker-compose.frontend.override.yml', '-p', config.get_prefix('frontend'), 'run', '--rm', 'kpi' ] # Make sure Postgres is running # We add this message to users because when AWS backups are activated, # it takes a long time to install the virtualenv in PostgreSQL # container, so the `wait_for_database` below sits there a while. # It makes us think kobo-install is frozen. CLI.colored_print( 'Waiting for PostgreSQL database to be up & running...', CLI.COLOR_INFO) frontend_command = kpi_run_command + _kpi_db_alias_kludge(' '.join( ['python', 'manage.py', 'wait_for_database', '--retries', '45'])) CLI.run_command(frontend_command, dict_['kobodocker_path']) CLI.colored_print('The PostgreSQL database is running!', CLI.COLOR_SUCCESS) frontend_command = kpi_run_command + _kpi_db_alias_kludge(' '.join( ['python', 'manage.py', 'is_database_empty', 'kpi', 'kobocat'])) output = CLI.run_command(frontend_command, dict_['kobodocker_path']) # TODO: read only stdout and don't consider stderr unless the exit code # is non-zero. Currently, `output` combines both stdout and stderr kpi_kc_db_empty = output.strip().split('\n')[-1] if kpi_kc_db_empty == 'True\tFalse': # KPI empty but KC is not: run the two-database upgrade script CLI.colored_print( 'Upgrading from single-database setup to separate databases ' 'for KPI and KoBoCAT', CLI.COLOR_INFO) message = ( 'Upgrading to separate databases is required to run the latest ' 'release of KoBoToolbox, but it may be a slow process if you ' 'have a lot of data. Expect at least one minute of downtime ' 'for every 1,500 KPI assets. Assets are surveys and library ' 'items: questions, blocks, and templates.\n' '\n' 'To postpone this process, downgrade to the last ' 'single-database release by stopping this script and executing ' 'the following commands:\n' '\n' ' python3 run.py --stop\n' ' git fetch\n' ' git checkout shared-database-obsolete\n' ' python3 run.py --update\n' ' python3 run.py --setup\n') CLI.framed_print(message) message = ( 'For help, visit https://community.kobotoolbox.org/t/upgrading-' 'to-separate-databases-for-kpi-and-kobocat/7202.') CLI.colored_print(message, CLI.COLOR_WARNING) response = CLI.yes_no_question('Do you want to proceed?', default=False) if response is False: sys.exit(0) backend_command = [ 'docker-compose', '-f', 'docker-compose.backend.{}.yml'.format(backend_role), '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), '-p', config.get_prefix('backend'), 'exec', 'postgres', 'bash', '/kobo-docker-scripts/primary/clone_data_from_kc_to_kpi.sh', '--noinput' ] try: subprocess.check_call(backend_command, cwd=dict_['kobodocker_path']) except subprocess.CalledProcessError: CLI.colored_print('An error has occurred', CLI.COLOR_ERROR) sys.exit(1) elif kpi_kc_db_empty not in [ 'True\tTrue', 'False\tTrue', 'False\tFalse', ]: # The output was invalid CLI.colored_print('An error has occurred', CLI.COLOR_ERROR) sys.stderr.write(kpi_kc_db_empty) sys.exit(1)
def info(cls, timeout=600): config = Config() dict_ = config.get_dict() nginx_port = dict_['exposed_nginx_docker_port'] main_url = '{}://{}.{}{}'.format( 'https' if dict_['https'] else 'http', dict_['kpi_subdomain'], dict_['public_domain_name'], ':{}'.format(nginx_port) if (nginx_port and str(nginx_port) != Config.DEFAULT_NGINX_PORT) else '') stop = False start = int(time.time()) success = False hostname = '{}.{}'.format(dict_['kpi_subdomain'], dict_['public_domain_name']) https = dict_['https'] nginx_port = int(Config.DEFAULT_NGINX_HTTPS_PORT) \ if https else int(dict_['exposed_nginx_docker_port']) already_retried = False while not stop: if Network.status_check(hostname, '/service_health/', nginx_port, https) == Network.STATUS_OK_200: stop = True success = True elif int(time.time()) - start >= timeout: if timeout > 0: CLI.colored_print( '\n`KoBoToolbox` has not started yet. ' 'This is can be normal with low CPU/RAM computers.\n', CLI.COLOR_INFO) question = 'Wait for another {} seconds?'.format(timeout) response = CLI.yes_no_question(question) if response: start = int(time.time()) continue else: if not already_retried: already_retried = True CLI.colored_print( '\nSometimes front-end containers cannot ' 'communicate with back-end containers.\n' 'Restarting the front-end containers usually ' 'fixes it.\n', CLI.COLOR_INFO) question = 'Would you like to try?' response = CLI.yes_no_question(question) if response: start = int(time.time()) cls.restart_frontend() continue stop = True else: sys.stdout.write('.') sys.stdout.flush() time.sleep(10) # Create a new line print('') if success: username = dict_['super_user_username'] password = dict_['super_user_password'] message = ('Ready\n' 'URL: {url}\n' 'User: {username}\n' 'Password: {password}').format(url=main_url, username=username, password=password) CLI.framed_print(message, color=CLI.COLOR_SUCCESS) else: message = ('KoBoToolbox could not start!\n' 'Please try `python3 run.py --logs` to see the logs.') CLI.framed_print(message, color=CLI.COLOR_ERROR) return success