def __questions_postgres_backups(self): """ Asks all questions about backups. """ self.__dict['use_backup'] = CLI.yes_no_question( 'Do you want to activate backups?', default=self.__dict['use_backup']) if self.__dict['use_backup']: self.__dict['use_wal_e'] = False schedule_regex_pattern = ( r'^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))' r'(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$') message = ('Schedules use linux cron syntax with UTC datetimes.\n' 'For example, schedule at 12:00 AM E.S.T every Sunday ' 'would be:\n' '0 5 * * 0\n' '\n' 'Please visit https://crontab.guru/ to generate a ' 'cron schedule.') CLI.colored_print('PostgreSQL backup cron expression?', CLI.COLOR_QUESTION) self.__dict['postgres_backup_schedule'] = CLI.get_response( '~{}'.format(schedule_regex_pattern), self.__dict['postgres_backup_schedule']) if self.aws: self.__questions_aws_backup_settings()
def logs(cls): config_object = Config() config = config_object.get_config() if config_object.primary_backend or config_object.secondary_backend: backend_role = config.get("backend_server_role", "primary") backend_command = [ "docker-compose", "-f", "docker-compose.backend.{}.yml".format(backend_role), "-f", "docker-compose.backend.{}.override.yml".format(backend_role), "-p", config_object.get_prefix("backend"), "logs", "-f" ] CLI.run_command(backend_command, config.get("kobodocker_path"), True) if config_object.frontend: frontend_command = [ "docker-compose", "-f", "docker-compose.frontend.yml", "-f", "docker-compose.frontend.override.yml", "-p", config_object.get_prefix("frontend"), "logs", "-f" ] CLI.run_command(frontend_command, config.get("kobodocker_path"), True)
def post_update(cls, cron): config = Config() # When `cron` is True, we want to bypass question and just recreate # YML and environment files from new templates if cron is True: current_dict = config.get_template() current_dict.update(config.get_dict()) config.set_config(current_dict) Template.render(config, force=True) sys.exit(0) message = ('After an update, it is strongly recommended to run\n' '`python3 run.py --setup` to regenerate environment files.') CLI.framed_print(message, color=CLI.COLOR_INFO) response = CLI.yes_no_question('Do you want to proceed?') if response is True: current_dict = config.build() Template.render(config) config.init_letsencrypt() Setup.update_hosts(current_dict) question = 'Do you want to (re)start containers?' response = CLI.yes_no_question(question) if response is True: Command.start()
def run(force_setup=False): if not platform.system() in ['Linux', 'Darwin']: CLI.colored_print('Not compatible with this OS', CLI.COLOR_ERROR) else: config = Config() dict_ = config.get_dict() if config.first_time: force_setup = True if force_setup: dict_ = config.build() Template.render(config) support = Support() support.copy_support_scripts() # # config.init_letsencrypt() # # Setup.update_hosts(dict_) else: print("Running smoothly") # if config.auto_detect_network(): # Template.render(config) # Setup.update_hosts(dict_) Command.start()
def run(force_setup=False): if sys.version_info[0] == 2: message = ( 'DEPRECATION: Python 2.7 has reached the end of its life on ' 'January 1st, 2020. Please upgrade your Python as Python 2.7 is ' 'not maintained anymore.\n\n' 'A future version of KoBoInstall will drop support for it.') CLI.framed_print(message) if not platform.system() in ['Linux', 'Darwin']: CLI.colored_print('Not compatible with this OS', CLI.COLOR_ERROR) else: config = Config() dict_ = config.get_dict() if config.first_time: force_setup = True if force_setup: dict_ = config.build() Setup.clone_kobodocker(config) Template.render(config) config.init_letsencrypt() Setup.update_hosts(dict_) else: if config.auto_detect_network(): Template.render(config) Setup.update_hosts(dict_) Command.start()
def run(cls, version=None, cron=False, update_self=True): # Validate kobo-docker already exists and is valid Setup.validate_already_run() if version is None: git_commit_version_command = [ 'git', 'rev-parse', '--abbrev-ref', 'HEAD', ] version = CLI.run_command(git_commit_version_command).strip() if update_self: # Update kobo-install first Setup.update_koboinstall(version) CLI.colored_print('kobo-install has been updated', CLI.COLOR_SUCCESS) # Reload this script to use `version`. # NB:`argv[0]` does not automatically get set to the executable # path as it usually would, so we have to do it manually--hence the # double `sys.executable` sys.argv.append(cls.NO_UPDATE_SELF_OPTION) os.execl(sys.executable, sys.executable, *sys.argv) # Update kobo-docker Setup.update_kobodocker() CLI.colored_print('kobo-docker has been updated', CLI.COLOR_SUCCESS) Setup.post_update(cron)
def logs(cls): config_object = Config() config = config_object.get_config() if config_object.master_backend or config_object.slave_backend: backend_role = config.get("backend_server_role", "master") backend_command = ["docker-compose", "-f", "docker-compose.backend.{}.yml".format(backend_role), "-f", "docker-compose.backend.{}.override.yml".format(backend_role), "logs", "-f"] if config.get("docker_prefix", "") != "": backend_command.insert(-2, "-p") backend_command.insert(-2, config.get("docker_prefix")) CLI.run_command(backend_command, config.get("kobodocker_path"), True) if config_object.frontend: frontend_command = ["docker-compose", "-f", "docker-compose.frontend.yml", "-f", "docker-compose.frontend.override.yml", "logs", "-f"] if config.get("docker_prefix", "") != "": frontend_command.insert(-2, "-p") frontend_command.insert(-2, config.get("docker_prefix")) CLI.run_command(frontend_command, config.get("kobodocker_path"), True)
def logs(cls): config = Config() dict_ = config.get_dict() if config.primary_backend or config.secondary_backend: backend_role = dict_['backend_server_role'] backend_command = ['docker-compose', '-f', 'docker-compose.backend.{}.yml'.format( backend_role), '-f', 'docker-compose.backend.{}.override.yml'.format( backend_role), '-p', config.get_prefix('backend'), 'logs', '-f' ] CLI.run_command(backend_command, dict_['kobodocker_path'], True) if config.frontend: frontend_command = ['docker-compose', '-f', 'docker-compose.frontend.yml', '-f', 'docker-compose.frontend.override.yml', '-p', config.get_prefix('frontend'), 'logs', '-f'] CLI.run_command(frontend_command, dict_['kobodocker_path'], True)
def clone_kobodocker(cls, config): """ Args: config (helpers.config.Config) """ dict_ = config.get_dict() do_update = config.first_time if not os.path.isdir(os.path.join(dict_['kobodocker_path'], '.git')): # Move unique id file to /tmp in order to clone without errors # (e.g. not empty directory) tmp_dirpath = tempfile.mkdtemp() shutil.move( os.path.join(dict_['kobodocker_path'], Config.UNIQUE_ID_FILE), os.path.join(tmp_dirpath, Config.UNIQUE_ID_FILE)) # clone project git_command = [ 'git', 'clone', 'https://github.com/kobotoolbox/kobo-docker', dict_['kobodocker_path'] ] CLI.run_command(git_command, cwd=os.path.dirname(dict_['kobodocker_path'])) shutil.move( os.path.join(tmp_dirpath, Config.UNIQUE_ID_FILE), os.path.join(dict_['kobodocker_path'], Config.UNIQUE_ID_FILE)) shutil.rmtree(tmp_dirpath) do_update = True # Force update if do_update: cls.update_kobodocker(dict_)
def read_unique_id(self): """ Reads unique id from file `Config.UNIQUE_ID_FILE` Returns: str """ unique_id = None try: unique_id_file = os.path.join(self.__dict['support_api_path'], Config.UNIQUE_ID_FILE) except KeyError: if self.first_time: return None else: CLI.framed_print( 'Bad configuration! The path of support_api ' 'path is missing. Please delete `.run.conf` ' 'and start from scratch', color=CLI.COLOR_ERROR) sys.exit(1) try: with open(unique_id_file, 'r') as f: unique_id = f.read().strip() except FileNotFoundError: pass return unique_id
def clone_kobodocker(cls, config_object): """ :param config_object: `Config` """ config = config_object.get_config() do_update = config_object.first_time if not os.path.isdir(os.path.join(config["kobodocker_path"], ".git")): # Move unique id file to /tmp in order to clone without errors # (e.g. not empty directory) tmp_dirpath = tempfile.mkdtemp() shutil.move( os.path.join(config["kobodocker_path"], Config.UNIQUE_ID_FILE), os.path.join(tmp_dirpath, Config.UNIQUE_ID_FILE)) # clone project git_command = [ "git", "clone", "https://github.com/kobotoolbox/kobo-docker", config["kobodocker_path"] ] CLI.run_command(git_command, cwd=os.path.dirname(config["kobodocker_path"])) shutil.move( os.path.join(tmp_dirpath, Config.UNIQUE_ID_FILE), os.path.join(config["kobodocker_path"], Config.UNIQUE_ID_FILE)) shutil.rmtree(tmp_dirpath) do_update = True # Force update if do_update: cls.update_kobodocker(config)
def __questions_kobo_postgres(self): """ KoBoToolbox's credentials """ # kobo_db_server self.__dict['kobo_db_server'] = CLI.colored_input( 'KoBoToolbox PostgreSQL server?', CLI.COLOR_QUESTION, self.__dict['kobo_db_server']) # kobo_db_name - Kobo Form CLI.colored_print('KoBoToolbox\'s KoboFORM PostgreSQL database name?', CLI.COLOR_QUESTION) self.__dict['kobo_db_name'] = CLI.get_response( r'~^\w+$', self.__dict['kobo_db_name'], to_lower=False) # kobo_cat_db_name - Kobo Form CLI.colored_print('KoBoToolbox\'s KoboCAT PostgreSQL database name?', CLI.COLOR_QUESTION) self.__dict['kobo_cat_db_name'] = CLI.get_response( r'~^\w+$', self.__dict['kobo_cat_db_name'], to_lower=False) # kobo_db_port self.__dict['kobo_db_port'] = CLI.colored_input( 'KoBoToolbox PostgreSQL Port?', CLI.COLOR_QUESTION, self.__dict['kobo_db_port']) # kobo_db_user self.__dict['kobo_db_user'] = CLI.colored_input( 'KoBoToolbox PostgreSQL User?', CLI.COLOR_QUESTION, self.__dict['kobo_db_user']) # kobo_db_password self.__dict['kobo_db_password'] = CLI.colored_input( 'KoBoToolbox PostgreSQL Password?', CLI.COLOR_QUESTION, self.__dict['kobo_db_password'])
def __create_directory(cls, environment_directory, path="", config={}, base_dir=""): if "docker-compose" in path: destination_directory = config.get("kobodocker_path") else: path = os.path.join( path, "") # Handle case when path is root and equals "". destination_directory = os.path.realpath( os.path.join( environment_directory, path.replace(os.path.join(base_dir, "templates", ""), ""))) if not os.path.isdir(destination_directory): try: os.makedirs(destination_directory) except OSError: CLI.colored_print( "Can not create {}. Please verify permissions!".format( destination_directory), CLI.COLOR_ERROR) sys.exit() return destination_directory
def version(cls): git_commit_version_command = ['git', 'rev-parse', 'HEAD'] stdout = CLI.run_command(git_commit_version_command) CLI.colored_print('kobo-install Version: {} (build {})'.format( Config.KOBO_INSTALL_VERSION, stdout.strip()[0:7], ), CLI.COLOR_SUCCESS)
def version(cls): git_commit_version_command = ["git", "rev-parse", "HEAD"] stdout = CLI.run_command(git_commit_version_command) CLI.colored_print( "KoBoInstall Version: {} (build {})".format( Config.KOBO_INSTALL_VERSION, stdout.strip()[0:7], ), CLI.COLOR_SUCCESS)
def build_image(image_): frontend_command = [ 'docker-compose', '-f', 'docker-compose.frontend.yml', '-f', 'docker-compose.frontend.override.yml', '-p', config.get_prefix('frontend'), 'build', '--force-rm', '--no-cache', image_ ] CLI.run_command(frontend_command, dict_['kobodocker_path'])
def render(cls, config, force=False): """ Write configuration files based on `config` Args: config (helpers.config.Config) force (bool) """ dict_ = config.get_dict() template_variables = cls.__get_template_variables(config) environment_directory = config.get_env_files_path() unique_id = cls.__read_unique_id(environment_directory) if (not force and unique_id and str(dict_.get('unique_id', '')) != str(unique_id)): message = ( 'WARNING!\n\n' 'Existing environment files are detected. Files will be ' 'overwritten.') CLI.framed_print(message) response = CLI.yes_no_question('Do you want to continue?', default=False) if not response: sys.exit(0) cls.__write_unique_id(environment_directory, dict_['unique_id']) base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) templates_path_parent = os.path.join(base_dir, 'templates') # Environment templates_path = os.path.join(templates_path_parent, Config.ENV_FILES_DIR, '') for root, dirnames, filenames in os.walk(templates_path): destination_directory = cls.__create_directory( environment_directory, root, templates_path) cls.__write_templates(template_variables, root, destination_directory, filenames) # kobo-docker templates_path = os.path.join(templates_path_parent, 'kobo-docker') for root, dirnames, filenames in os.walk(templates_path): destination_directory = dict_['kobodocker_path'] cls.__write_templates(template_variables, root, destination_directory, filenames) # nginx-certbox if config.use_letsencrypt: templates_path = os.path.join(templates_path_parent, Config.LETSENCRYPT_DOCKER_DIR, '') for root, dirnames, filenames in os.walk(templates_path): destination_directory = cls.__create_directory( config.get_letsencrypt_repo_path(), root, templates_path) cls.__write_templates(template_variables, root, destination_directory, filenames)
def build_image(image_): frontend_command = [ "docker-compose", "-f", "docker-compose.frontend.yml", "-f", "docker-compose.frontend.override.yml", "-p", config_object.get_prefix("frontend"), "build", "--force-rm", "--no-cache", image_ ] CLI.run_command(frontend_command, config.get("kobodocker_path"))
def stop_nginx(cls): config_object = Config() config = config_object.get_config() nginx_stop_command = [ "docker-compose", "-f", "docker-compose.frontend.yml", "-f", "docker-compose.frontend.override.yml", "-p", config_object.get_prefix("frontend"), "stop", "nginx" ] CLI.run_command(nginx_stop_command, config.get("kobodocker_path"))
def stop(cls, output=True, frontend_only=False): """ Stop containers """ config = Config() dict_ = config.get_dict() if not config.multi_servers or config.frontend: # Shut down maintenance container in case it's up&running maintenance_down_command = [ 'docker-compose', '-f', 'docker-compose.maintenance.yml', '-f', 'docker-compose.maintenance.override.yml', '-p', config.get_prefix('maintenance'), 'down' ] CLI.run_command(maintenance_down_command, dict_['kobodocker_path']) # Stop reverse proxy if user uses it. if config.use_letsencrypt: proxy_command = ['docker-compose', 'down'] CLI.run_command(proxy_command, config.get_letsencrypt_repo_path()) # Shut down front-end containers frontend_command = [ 'docker-compose', '-f', 'docker-compose.frontend.yml', '-f', 'docker-compose.frontend.override.yml', '-p', config.get_prefix('frontend'), 'down', ] cls.__validate_custom_yml(config, frontend_command) CLI.run_command(frontend_command, dict_['kobodocker_path']) if not frontend_only and config.backend: backend_role = dict_['backend_server_role'] backend_command = [ 'docker-compose', '-f', 'docker-compose.backend.{}.yml'.format(backend_role), '-f', 'docker-compose.backend.{}.override.yml'.format(backend_role), '-p', config.get_prefix('backend'), 'down' ] cls.__validate_custom_yml(config, backend_command) CLI.run_command(backend_command, dict_['kobodocker_path']) if output: CLI.colored_print('KoboToolbox has been stopped', CLI.COLOR_SUCCESS)
def stop_nginx(cls): config = Config() dict_ = config.get_dict() nginx_stop_command = [ 'docker-compose', '-f', 'docker-compose.frontend.yml', '-f', 'docker-compose.frontend.override.yml', '-p', config.get_prefix('frontend'), 'stop', 'nginx' ] CLI.run_command(nginx_stop_command, dict_['kobodocker_path'])
def start_maintenance(cls): config_object = Config() config = config_object.get_config() frontend_command = [ "docker-compose", "-f", "docker-compose.maintenance.yml", "-p", config_object.get_prefix("maintenance"), "up", "-d", "maintenance" ] CLI.run_command(frontend_command, config.get("kobodocker_path")) CLI.colored_print("Maintenance mode has been started", CLI.COLOR_SUCCESS)
def __welcome(): message = ( 'Welcome to SUPPORT API for KoBoToolbox.\n' '\n' 'You are going to be asked some questions that will determine how ' 'to build the configuration of `Support API`.\n' '\n' 'Some questions already have default values (within brackets).\n' 'Just press `enter` to accept the default value or enter `-` to ' 'remove previously entered value.\n' 'Otherwise choose between choices or type your answer. ') CLI.framed_print(message, color=CLI.COLOR_INFO)
def __questions_aws_configuration(self): if self.__dict['use_aws']: self.__dict['aws_access_key'] = CLI.colored_input( 'AWS Access Key', CLI.COLOR_QUESTION, self.__dict['aws_access_key']) self.__dict['aws_secret_key'] = CLI.colored_input( 'AWS Secret Key', CLI.COLOR_QUESTION, self.__dict['aws_secret_key']) else: self.__dict['aws_access_key'] = '' self.__dict['aws_secret_key'] = ''
def build_image(image_): frontend_command = ["docker-compose", "-f", "docker-compose.frontend.yml", "-f", "docker-compose.frontend.override.yml", "build", "--force-rm", "--no-cache", image_] if config.get("docker_prefix", "") != "": frontend_command.insert(-4, "-p") frontend_command.insert(-4, config.get("docker_prefix")) CLI.run_command(frontend_command, config.get("kobodocker_path"))
def start_maintenance(cls): config = Config() dict_ = config.get_dict() frontend_command = [ 'docker-compose', '-f', 'docker-compose.maintenance.yml', '-f', 'docker-compose.maintenance.override.yml', '-p', config.get_prefix('maintenance'), 'up', '-d' ] CLI.run_command(frontend_command, dict_['kobodocker_path']) CLI.colored_print('Maintenance mode has been started', CLI.COLOR_SUCCESS)
def __validate_custom_yml(config, command): """ Validate whether docker-compose must start the containers with a custom YML file in addition to the default. If the file does not yet exist, kobo-install is paused until the user creates it and resumes the setup manually. If user has chosen to use a custom YML file, it is injected into `command` before being executed. """ dict_ = config.get_dict() frontend_command = True # Detect if it's a front-end command or back-end command for part in command: if 'backend' in part: frontend_command = False break if frontend_command and dict_['use_frontend_custom_yml']: custom_file = '{}/docker-compose.frontend.custom.yml'.format( dict_['kobodocker_path']) does_custom_file_exist = os.path.exists(custom_file) while not does_custom_file_exist: message = ('Please create your custom configuration in\n' '`{custom_file}`.').format(custom_file=custom_file) CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90) input('Press any key when it is done...') does_custom_file_exist = os.path.exists(custom_file) # Add custom file to docker-compose command command.insert(5, '-f') command.insert(6, 'docker-compose.frontend.custom.yml') if not frontend_command and dict_['use_backend_custom_yml']: backend_server_role = dict_['backend_server_role'] custom_file = '{}/docker-compose.backend.{}.custom.yml'.format( dict_['kobodocker_path'], backend_server_role) does_custom_file_exist = os.path.exists(custom_file) while not does_custom_file_exist: message = ('Please create your custom configuration in\n' '`{custom_file}`.').format(custom_file=custom_file) CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90) input('Press any key when it is done...') does_custom_file_exist = os.path.exists(custom_file) # Add custom file to docker-compose command command.insert(5, '-f') command.insert( 6, 'docker-compose.backend.{}.custom.yml'.format( backend_server_role))
def write_unique_id(self): try: unique_id_file = os.path.join(self.__dict['support_api_path'], Config.UNIQUE_ID_FILE) with open(unique_id_file, 'w') as f: f.write(str(self.__dict['unique_id'])) os.chmod(unique_id_file, stat.S_IWRITE | stat.S_IREAD) except (IOError, OSError): CLI.colored_print('Could not write unique_id file', CLI.COLOR_ERROR) return False return True
def build(cls, image=None): """ Builds kpi/kobocat images with `--no-caches` option Pulls latest `kobotoolbox/koboform_base` as well :param image: str """ config_object = Config() config = config_object.get_config() if config_object.dev_mode or config_object.staging_mode: def build_image(image_): frontend_command = ["docker-compose", "-f", "docker-compose.frontend.yml", "-f", "docker-compose.frontend.override.yml", "build", "--force-rm", "--no-cache", image_] if config.get("docker_prefix", "") != "": frontend_command.insert(-4, "-p") frontend_command.insert(-4, config.get("docker_prefix")) CLI.run_command(frontend_command, config.get("kobodocker_path")) pull_base_command = ["docker", "pull", "kobotoolbox/koboform_base"] CLI.run_command(pull_base_command, config.get("kobodocker_path")) if image is None or image == "kf": config["kpi_dev_build_id"] = "{prefix}{timestamp}".format( prefix="{}.".format(config.get("docker_prefix")) if config.get("docker_prefix") else "", timestamp=str(int(time.time())) ) config_object.write_config() Template.render(config_object) build_image("kpi") if image is None or image == "kc": config["kc_dev_build_id"] = "{prefix}{timestamp}".format( prefix="{}.".format(config.get("docker_prefix")) if config.get("docker_prefix") else "", timestamp=str(int(time.time())) ) config_object.write_config() Template.render(config_object) build_image("kobocat")
def __create_directory(self): """ Create repository directory if it doesn't exist. """ CLI.colored_print('Where do you want to install?', CLI.COLOR_QUESTION) while True: support_api_path = CLI.colored_input( '', CLI.COLOR_QUESTION, self.__dict['support_api_path']) if support_api_path.startswith('.'): base_dir = os.path.dirname( os.path.dirname(os.path.realpath(__file__))) support_api_path = os.path.normpath( os.path.join(base_dir, support_api_path)) question = 'Please confirm path [{}]'.format(support_api_path) response = CLI.yes_no_question(question) if response is True: if os.path.isdir(support_api_path): break else: try: os.makedirs(support_api_path) break except OSError: CLI.colored_print( 'Could not create directory {}!'.format( support_api_path), CLI.COLOR_ERROR) CLI.colored_print( 'Please make sure you have permissions ' 'and path is correct', CLI.COLOR_ERROR) self.__dict['support_api_path'] = support_api_path self.write_unique_id() self.__validate_installation()