def __questions_postgres_backups(self):
"""
Asks all questions about backups.
"""
self.__dict['use_backup'] = CLI.yes_no_question(
'Do you want to activate backups?',
default=self.__dict['use_backup'])
if self.__dict['use_backup']:
self.__dict['use_wal_e'] = False
schedule_regex_pattern = (
r'^((((\d+(,\d+)*)|(\d+-\d+)|(\*(\/\d+)?)))'
r'(\s+(((\d+(,\d+)*)|(\d+\-\d+)|(\*(\/\d+)?)))){4})$')
message = ('Schedules use linux cron syntax with UTC datetimes.\n'
'For example, schedule at 12:00 AM E.S.T every Sunday '
'would be:\n'
'0 5 * * 0\n'
'\n'
'Please visit https://crontab.guru/ to generate a '
'cron schedule.')
CLI.colored_print('PostgreSQL backup cron expression?',
CLI.COLOR_QUESTION)
self.__dict['postgres_backup_schedule'] = CLI.get_response(
'~{}'.format(schedule_regex_pattern),
self.__dict['postgres_backup_schedule'])
if self.aws:
self.__questions_aws_backup_settings()
def logs(cls):
config_object = Config()
config = config_object.get_config()
if config_object.primary_backend or config_object.secondary_backend:
backend_role = config.get("backend_server_role", "primary")
backend_command = [
"docker-compose", "-f",
"docker-compose.backend.{}.yml".format(backend_role), "-f",
"docker-compose.backend.{}.override.yml".format(backend_role),
"-p",
config_object.get_prefix("backend"), "logs", "-f"
]
CLI.run_command(backend_command, config.get("kobodocker_path"),
True)
if config_object.frontend:
frontend_command = [
"docker-compose", "-f", "docker-compose.frontend.yml", "-f",
"docker-compose.frontend.override.yml", "-p",
config_object.get_prefix("frontend"), "logs", "-f"
]
CLI.run_command(frontend_command, config.get("kobodocker_path"),
True)
def post_update(cls, cron):
config = Config()
# When `cron` is True, we want to bypass question and just recreate
# YML and environment files from new templates
if cron is True:
current_dict = config.get_template()
current_dict.update(config.get_dict())
config.set_config(current_dict)
Template.render(config, force=True)
sys.exit(0)
message = ('After an update, it is strongly recommended to run\n'
'`python3 run.py --setup` to regenerate environment files.')
CLI.framed_print(message, color=CLI.COLOR_INFO)
response = CLI.yes_no_question('Do you want to proceed?')
if response is True:
current_dict = config.build()
Template.render(config)
config.init_letsencrypt()
Setup.update_hosts(current_dict)
question = 'Do you want to (re)start containers?'
response = CLI.yes_no_question(question)
if response is True:
Command.start()
def run(force_setup=False):
if not platform.system() in ['Linux', 'Darwin']:
CLI.colored_print('Not compatible with this OS', CLI.COLOR_ERROR)
else:
config = Config()
dict_ = config.get_dict()
if config.first_time:
force_setup = True
if force_setup:
dict_ = config.build()
Template.render(config)
support = Support()
support.copy_support_scripts()
# # config.init_letsencrypt()
# # Setup.update_hosts(dict_)
else:
print("Running smoothly")
# if config.auto_detect_network():
# Template.render(config)
# Setup.update_hosts(dict_)
Command.start()
def run(force_setup=False):
if sys.version_info[0] == 2:
message = (
'DEPRECATION: Python 2.7 has reached the end of its life on '
'January 1st, 2020. Please upgrade your Python as Python 2.7 is '
'not maintained anymore.\n\n'
'A future version of KoBoInstall will drop support for it.')
CLI.framed_print(message)
if not platform.system() in ['Linux', 'Darwin']:
CLI.colored_print('Not compatible with this OS', CLI.COLOR_ERROR)
else:
config = Config()
dict_ = config.get_dict()
if config.first_time:
force_setup = True
if force_setup:
dict_ = config.build()
Setup.clone_kobodocker(config)
Template.render(config)
config.init_letsencrypt()
Setup.update_hosts(dict_)
else:
if config.auto_detect_network():
Template.render(config)
Setup.update_hosts(dict_)
Command.start()
def run(cls, version=None, cron=False, update_self=True):
# Validate kobo-docker already exists and is valid
Setup.validate_already_run()
if version is None:
git_commit_version_command = [
'git',
'rev-parse',
'--abbrev-ref',
'HEAD',
]
version = CLI.run_command(git_commit_version_command).strip()
if update_self:
# Update kobo-install first
Setup.update_koboinstall(version)
CLI.colored_print('kobo-install has been updated',
CLI.COLOR_SUCCESS)
# Reload this script to use `version`.
# NB:`argv[0]` does not automatically get set to the executable
# path as it usually would, so we have to do it manually--hence the
# double `sys.executable`
sys.argv.append(cls.NO_UPDATE_SELF_OPTION)
os.execl(sys.executable, sys.executable, *sys.argv)
# Update kobo-docker
Setup.update_kobodocker()
CLI.colored_print('kobo-docker has been updated', CLI.COLOR_SUCCESS)
Setup.post_update(cron)
def logs(cls):
config_object = Config()
config = config_object.get_config()
if config_object.master_backend or config_object.slave_backend:
backend_role = config.get("backend_server_role", "master")
backend_command = ["docker-compose",
"-f", "docker-compose.backend.{}.yml".format(backend_role),
"-f", "docker-compose.backend.{}.override.yml".format(backend_role),
"logs", "-f"]
if config.get("docker_prefix", "") != "":
backend_command.insert(-2, "-p")
backend_command.insert(-2, config.get("docker_prefix"))
CLI.run_command(backend_command, config.get("kobodocker_path"), True)
if config_object.frontend:
frontend_command = ["docker-compose",
"-f", "docker-compose.frontend.yml",
"-f", "docker-compose.frontend.override.yml",
"logs", "-f"]
if config.get("docker_prefix", "") != "":
frontend_command.insert(-2, "-p")
frontend_command.insert(-2, config.get("docker_prefix"))
CLI.run_command(frontend_command, config.get("kobodocker_path"), True)
def logs(cls):
config = Config()
dict_ = config.get_dict()
if config.primary_backend or config.secondary_backend:
backend_role = dict_['backend_server_role']
backend_command = ['docker-compose',
'-f',
'docker-compose.backend.{}.yml'.format(
backend_role),
'-f',
'docker-compose.backend.{}.override.yml'.format(
backend_role),
'-p',
config.get_prefix('backend'),
'logs',
'-f'
]
CLI.run_command(backend_command,
dict_['kobodocker_path'],
True)
if config.frontend:
frontend_command = ['docker-compose',
'-f', 'docker-compose.frontend.yml',
'-f', 'docker-compose.frontend.override.yml',
'-p', config.get_prefix('frontend'),
'logs', '-f']
CLI.run_command(frontend_command,
dict_['kobodocker_path'],
True)
def clone_kobodocker(cls, config):
"""
Args:
config (helpers.config.Config)
"""
dict_ = config.get_dict()
do_update = config.first_time
if not os.path.isdir(os.path.join(dict_['kobodocker_path'], '.git')):
# Move unique id file to /tmp in order to clone without errors
# (e.g. not empty directory)
tmp_dirpath = tempfile.mkdtemp()
shutil.move(
os.path.join(dict_['kobodocker_path'], Config.UNIQUE_ID_FILE),
os.path.join(tmp_dirpath, Config.UNIQUE_ID_FILE))
# clone project
git_command = [
'git', 'clone', 'https://github.com/kobotoolbox/kobo-docker',
dict_['kobodocker_path']
]
CLI.run_command(git_command,
cwd=os.path.dirname(dict_['kobodocker_path']))
shutil.move(
os.path.join(tmp_dirpath, Config.UNIQUE_ID_FILE),
os.path.join(dict_['kobodocker_path'], Config.UNIQUE_ID_FILE))
shutil.rmtree(tmp_dirpath)
do_update = True # Force update
if do_update:
cls.update_kobodocker(dict_)
def read_unique_id(self):
"""
Reads unique id from file `Config.UNIQUE_ID_FILE`
Returns:
str
"""
unique_id = None
try:
unique_id_file = os.path.join(self.__dict['support_api_path'],
Config.UNIQUE_ID_FILE)
except KeyError:
if self.first_time:
return None
else:
CLI.framed_print(
'Bad configuration! The path of support_api '
'path is missing. Please delete `.run.conf` '
'and start from scratch',
color=CLI.COLOR_ERROR)
sys.exit(1)
try:
with open(unique_id_file, 'r') as f:
unique_id = f.read().strip()
except FileNotFoundError:
pass
return unique_id
def clone_kobodocker(cls, config_object):
"""
:param config_object: `Config`
"""
config = config_object.get_config()
do_update = config_object.first_time
if not os.path.isdir(os.path.join(config["kobodocker_path"], ".git")):
# Move unique id file to /tmp in order to clone without errors
# (e.g. not empty directory)
tmp_dirpath = tempfile.mkdtemp()
shutil.move(
os.path.join(config["kobodocker_path"], Config.UNIQUE_ID_FILE),
os.path.join(tmp_dirpath, Config.UNIQUE_ID_FILE))
# clone project
git_command = [
"git", "clone", "https://github.com/kobotoolbox/kobo-docker",
config["kobodocker_path"]
]
CLI.run_command(git_command,
cwd=os.path.dirname(config["kobodocker_path"]))
shutil.move(
os.path.join(tmp_dirpath, Config.UNIQUE_ID_FILE),
os.path.join(config["kobodocker_path"], Config.UNIQUE_ID_FILE))
shutil.rmtree(tmp_dirpath)
do_update = True # Force update
if do_update:
cls.update_kobodocker(config)
def __questions_kobo_postgres(self):
"""
KoBoToolbox's credentials
"""
# kobo_db_server
self.__dict['kobo_db_server'] = CLI.colored_input(
'KoBoToolbox PostgreSQL server?', CLI.COLOR_QUESTION,
self.__dict['kobo_db_server'])
# kobo_db_name - Kobo Form
CLI.colored_print('KoBoToolbox\'s KoboFORM PostgreSQL database name?',
CLI.COLOR_QUESTION)
self.__dict['kobo_db_name'] = CLI.get_response(
r'~^\w+$', self.__dict['kobo_db_name'], to_lower=False)
# kobo_cat_db_name - Kobo Form
CLI.colored_print('KoBoToolbox\'s KoboCAT PostgreSQL database name?',
CLI.COLOR_QUESTION)
self.__dict['kobo_cat_db_name'] = CLI.get_response(
r'~^\w+$', self.__dict['kobo_cat_db_name'], to_lower=False)
# kobo_db_port
self.__dict['kobo_db_port'] = CLI.colored_input(
'KoBoToolbox PostgreSQL Port?', CLI.COLOR_QUESTION,
self.__dict['kobo_db_port'])
# kobo_db_user
self.__dict['kobo_db_user'] = CLI.colored_input(
'KoBoToolbox PostgreSQL User?', CLI.COLOR_QUESTION,
self.__dict['kobo_db_user'])
# kobo_db_password
self.__dict['kobo_db_password'] = CLI.colored_input(
'KoBoToolbox PostgreSQL Password?', CLI.COLOR_QUESTION,
self.__dict['kobo_db_password'])
def __create_directory(cls,
environment_directory,
path="",
config={},
base_dir=""):
if "docker-compose" in path:
destination_directory = config.get("kobodocker_path")
else:
path = os.path.join(
path, "") # Handle case when path is root and equals "".
destination_directory = os.path.realpath(
os.path.join(
environment_directory,
path.replace(os.path.join(base_dir, "templates", ""), "")))
if not os.path.isdir(destination_directory):
try:
os.makedirs(destination_directory)
except OSError:
CLI.colored_print(
"Can not create {}. Please verify permissions!".format(
destination_directory), CLI.COLOR_ERROR)
sys.exit()
return destination_directory
def version(cls):
git_commit_version_command = ['git', 'rev-parse', 'HEAD']
stdout = CLI.run_command(git_commit_version_command)
CLI.colored_print('kobo-install Version: {} (build {})'.format(
Config.KOBO_INSTALL_VERSION,
stdout.strip()[0:7],
), CLI.COLOR_SUCCESS)
def version(cls):
git_commit_version_command = ["git", "rev-parse", "HEAD"]
stdout = CLI.run_command(git_commit_version_command)
CLI.colored_print(
"KoBoInstall Version: {} (build {})".format(
Config.KOBO_INSTALL_VERSION,
stdout.strip()[0:7],
), CLI.COLOR_SUCCESS)
def build_image(image_):
frontend_command = [
'docker-compose', '-f', 'docker-compose.frontend.yml',
'-f', 'docker-compose.frontend.override.yml', '-p',
config.get_prefix('frontend'), 'build', '--force-rm',
'--no-cache', image_
]
CLI.run_command(frontend_command, dict_['kobodocker_path'])
def render(cls, config, force=False):
"""
Write configuration files based on `config`
Args:
config (helpers.config.Config)
force (bool)
"""
dict_ = config.get_dict()
template_variables = cls.__get_template_variables(config)
environment_directory = config.get_env_files_path()
unique_id = cls.__read_unique_id(environment_directory)
if (not force and unique_id
and str(dict_.get('unique_id', '')) != str(unique_id)):
message = (
'WARNING!\n\n'
'Existing environment files are detected. Files will be '
'overwritten.')
CLI.framed_print(message)
response = CLI.yes_no_question('Do you want to continue?',
default=False)
if not response:
sys.exit(0)
cls.__write_unique_id(environment_directory, dict_['unique_id'])
base_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
templates_path_parent = os.path.join(base_dir, 'templates')
# Environment
templates_path = os.path.join(templates_path_parent,
Config.ENV_FILES_DIR, '')
for root, dirnames, filenames in os.walk(templates_path):
destination_directory = cls.__create_directory(
environment_directory, root, templates_path)
cls.__write_templates(template_variables, root,
destination_directory, filenames)
# kobo-docker
templates_path = os.path.join(templates_path_parent, 'kobo-docker')
for root, dirnames, filenames in os.walk(templates_path):
destination_directory = dict_['kobodocker_path']
cls.__write_templates(template_variables, root,
destination_directory, filenames)
# nginx-certbox
if config.use_letsencrypt:
templates_path = os.path.join(templates_path_parent,
Config.LETSENCRYPT_DOCKER_DIR, '')
for root, dirnames, filenames in os.walk(templates_path):
destination_directory = cls.__create_directory(
config.get_letsencrypt_repo_path(), root, templates_path)
cls.__write_templates(template_variables, root,
destination_directory, filenames)
def build_image(image_):
frontend_command = [
"docker-compose", "-f", "docker-compose.frontend.yml",
"-f", "docker-compose.frontend.override.yml", "-p",
config_object.get_prefix("frontend"), "build",
"--force-rm", "--no-cache", image_
]
CLI.run_command(frontend_command,
config.get("kobodocker_path"))
def stop_nginx(cls):
config_object = Config()
config = config_object.get_config()
nginx_stop_command = [
"docker-compose", "-f", "docker-compose.frontend.yml", "-f",
"docker-compose.frontend.override.yml", "-p",
config_object.get_prefix("frontend"), "stop", "nginx"
]
CLI.run_command(nginx_stop_command, config.get("kobodocker_path"))
def stop(cls, output=True, frontend_only=False):
"""
Stop containers
"""
config = Config()
dict_ = config.get_dict()
if not config.multi_servers or config.frontend:
# Shut down maintenance container in case it's up&running
maintenance_down_command = [
'docker-compose', '-f', 'docker-compose.maintenance.yml', '-f',
'docker-compose.maintenance.override.yml', '-p',
config.get_prefix('maintenance'), 'down'
]
CLI.run_command(maintenance_down_command, dict_['kobodocker_path'])
# Stop reverse proxy if user uses it.
if config.use_letsencrypt:
proxy_command = ['docker-compose', 'down']
CLI.run_command(proxy_command,
config.get_letsencrypt_repo_path())
# Shut down front-end containers
frontend_command = [
'docker-compose',
'-f',
'docker-compose.frontend.yml',
'-f',
'docker-compose.frontend.override.yml',
'-p',
config.get_prefix('frontend'),
'down',
]
cls.__validate_custom_yml(config, frontend_command)
CLI.run_command(frontend_command, dict_['kobodocker_path'])
if not frontend_only and config.backend:
backend_role = dict_['backend_server_role']
backend_command = [
'docker-compose', '-f',
'docker-compose.backend.{}.yml'.format(backend_role), '-f',
'docker-compose.backend.{}.override.yml'.format(backend_role),
'-p',
config.get_prefix('backend'), 'down'
]
cls.__validate_custom_yml(config, backend_command)
CLI.run_command(backend_command, dict_['kobodocker_path'])
if output:
CLI.colored_print('KoboToolbox has been stopped',
CLI.COLOR_SUCCESS)
def stop_nginx(cls):
config = Config()
dict_ = config.get_dict()
nginx_stop_command = [
'docker-compose', '-f', 'docker-compose.frontend.yml', '-f',
'docker-compose.frontend.override.yml', '-p',
config.get_prefix('frontend'), 'stop', 'nginx'
]
CLI.run_command(nginx_stop_command, dict_['kobodocker_path'])
def start_maintenance(cls):
config_object = Config()
config = config_object.get_config()
frontend_command = [
"docker-compose", "-f", "docker-compose.maintenance.yml", "-p",
config_object.get_prefix("maintenance"), "up", "-d", "maintenance"
]
CLI.run_command(frontend_command, config.get("kobodocker_path"))
CLI.colored_print("Maintenance mode has been started",
CLI.COLOR_SUCCESS)
def __welcome():
message = (
'Welcome to SUPPORT API for KoBoToolbox.\n'
'\n'
'You are going to be asked some questions that will determine how '
'to build the configuration of `Support API`.\n'
'\n'
'Some questions already have default values (within brackets).\n'
'Just press `enter` to accept the default value or enter `-` to '
'remove previously entered value.\n'
'Otherwise choose between choices or type your answer. ')
CLI.framed_print(message, color=CLI.COLOR_INFO)
def __questions_aws_configuration(self):
if self.__dict['use_aws']:
self.__dict['aws_access_key'] = CLI.colored_input(
'AWS Access Key', CLI.COLOR_QUESTION,
self.__dict['aws_access_key'])
self.__dict['aws_secret_key'] = CLI.colored_input(
'AWS Secret Key', CLI.COLOR_QUESTION,
self.__dict['aws_secret_key'])
else:
self.__dict['aws_access_key'] = ''
self.__dict['aws_secret_key'] = ''
def build_image(image_):
frontend_command = ["docker-compose",
"-f", "docker-compose.frontend.yml",
"-f", "docker-compose.frontend.override.yml",
"build", "--force-rm", "--no-cache",
image_]
if config.get("docker_prefix", "") != "":
frontend_command.insert(-4, "-p")
frontend_command.insert(-4, config.get("docker_prefix"))
CLI.run_command(frontend_command, config.get("kobodocker_path"))
def start_maintenance(cls):
config = Config()
dict_ = config.get_dict()
frontend_command = [
'docker-compose', '-f', 'docker-compose.maintenance.yml', '-f',
'docker-compose.maintenance.override.yml', '-p',
config.get_prefix('maintenance'), 'up', '-d'
]
CLI.run_command(frontend_command, dict_['kobodocker_path'])
CLI.colored_print('Maintenance mode has been started',
CLI.COLOR_SUCCESS)
def __validate_custom_yml(config, command):
"""
Validate whether docker-compose must start the containers with a
custom YML file in addition to the default. If the file does not yet exist,
kobo-install is paused until the user creates it and resumes the setup manually.
If user has chosen to use a custom YML file, it is injected into `command`
before being executed.
"""
dict_ = config.get_dict()
frontend_command = True
# Detect if it's a front-end command or back-end command
for part in command:
if 'backend' in part:
frontend_command = False
break
if frontend_command and dict_['use_frontend_custom_yml']:
custom_file = '{}/docker-compose.frontend.custom.yml'.format(
dict_['kobodocker_path'])
does_custom_file_exist = os.path.exists(custom_file)
while not does_custom_file_exist:
message = ('Please create your custom configuration in\n'
'`{custom_file}`.').format(custom_file=custom_file)
CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90)
input('Press any key when it is done...')
does_custom_file_exist = os.path.exists(custom_file)
# Add custom file to docker-compose command
command.insert(5, '-f')
command.insert(6, 'docker-compose.frontend.custom.yml')
if not frontend_command and dict_['use_backend_custom_yml']:
backend_server_role = dict_['backend_server_role']
custom_file = '{}/docker-compose.backend.{}.custom.yml'.format(
dict_['kobodocker_path'], backend_server_role)
does_custom_file_exist = os.path.exists(custom_file)
while not does_custom_file_exist:
message = ('Please create your custom configuration in\n'
'`{custom_file}`.').format(custom_file=custom_file)
CLI.framed_print(message, color=CLI.COLOR_INFO, columns=90)
input('Press any key when it is done...')
does_custom_file_exist = os.path.exists(custom_file)
# Add custom file to docker-compose command
command.insert(5, '-f')
command.insert(
6, 'docker-compose.backend.{}.custom.yml'.format(
backend_server_role))
def write_unique_id(self):
try:
unique_id_file = os.path.join(self.__dict['support_api_path'],
Config.UNIQUE_ID_FILE)
with open(unique_id_file, 'w') as f:
f.write(str(self.__dict['unique_id']))
os.chmod(unique_id_file, stat.S_IWRITE | stat.S_IREAD)
except (IOError, OSError):
CLI.colored_print('Could not write unique_id file',
CLI.COLOR_ERROR)
return False
return True
def build(cls, image=None):
"""
Builds kpi/kobocat images with `--no-caches` option
Pulls latest `kobotoolbox/koboform_base` as well
:param image: str
"""
config_object = Config()
config = config_object.get_config()
if config_object.dev_mode or config_object.staging_mode:
def build_image(image_):
frontend_command = ["docker-compose",
"-f", "docker-compose.frontend.yml",
"-f", "docker-compose.frontend.override.yml",
"build", "--force-rm", "--no-cache",
image_]
if config.get("docker_prefix", "") != "":
frontend_command.insert(-4, "-p")
frontend_command.insert(-4, config.get("docker_prefix"))
CLI.run_command(frontend_command, config.get("kobodocker_path"))
pull_base_command = ["docker",
"pull",
"kobotoolbox/koboform_base"]
CLI.run_command(pull_base_command, config.get("kobodocker_path"))
if image is None or image == "kf":
config["kpi_dev_build_id"] = "{prefix}{timestamp}".format(
prefix="{}.".format(config.get("docker_prefix"))
if config.get("docker_prefix") else "",
timestamp=str(int(time.time()))
)
config_object.write_config()
Template.render(config_object)
build_image("kpi")
if image is None or image == "kc":
config["kc_dev_build_id"] = "{prefix}{timestamp}".format(
prefix="{}.".format(config.get("docker_prefix"))
if config.get("docker_prefix") else "",
timestamp=str(int(time.time()))
)
config_object.write_config()
Template.render(config_object)
build_image("kobocat")
def __create_directory(self):
"""
Create repository directory if it doesn't exist.
"""
CLI.colored_print('Where do you want to install?', CLI.COLOR_QUESTION)
while True:
support_api_path = CLI.colored_input(
'', CLI.COLOR_QUESTION, self.__dict['support_api_path'])
if support_api_path.startswith('.'):
base_dir = os.path.dirname(
os.path.dirname(os.path.realpath(__file__)))
support_api_path = os.path.normpath(
os.path.join(base_dir, support_api_path))
question = 'Please confirm path [{}]'.format(support_api_path)
response = CLI.yes_no_question(question)
if response is True:
if os.path.isdir(support_api_path):
break
else:
try:
os.makedirs(support_api_path)
break
except OSError:
CLI.colored_print(
'Could not create directory {}!'.format(
support_api_path), CLI.COLOR_ERROR)
CLI.colored_print(
'Please make sure you have permissions '
'and path is correct', CLI.COLOR_ERROR)
self.__dict['support_api_path'] = support_api_path
self.write_unique_id()
self.__validate_installation()
