def validate(self):
with open(self.options['config'], 'r') as f:
self.config = json.load(f)
self.config['ROOT_DIR'] = os.getcwd()
missing = []
for item in self.validation_config:
key = item['key']
required = item['required']
key_found = key in self.config or key in os.environ
if not self.config.get(key):
self.config[key] = os.environ.get(key, "")
if (not key_found and required) or (key_found and required
and not self.config[key]):
missing.append(item)
elif not required and not self.config[key]:
warning = f"Config missing optional field: {key} - {Colors.WARNING}{item['description']}"
if item.get('default-message'):
warning += f" - {Colors.CYAN}{item['default-message']}"
Prompt.notice(warning)
if missing:
missing_formatted = [
f"{x['key']}: {x['description']}" for x in missing
]
Prompt.error(
f"The following keys are missing/empty from your env or config file: {missing_formatted}",
close=True)
# Prepares build files if provided
super().docker_tmp_file_handler(self.config, self.TMP_BUILD_FILES)
def signal_handler(self, sig, frame):
Prompt.notice("\nCtrl-c captured. Executing teardown function.")
if not self.kill_captured:
self.kill_captured = True
self.cleanup()
self.on_sig_kill()
sys.exit(0)
def run(self):
self.config.setdefault('PERSIST_STACK')
if not self.config['PERSIST_STACK']:
self.FAIL_SUFFIX = "--remove-orphans --rmi all"
del self.config['PERSIST_STACK']
self.check_if_docker_is_started()
self.set_default('COMPOSE_PROJECT_NAME',
'govready-q') # Prefix for all docker containers
self.set_default('GIT_URL',
"https://github.com/GovReady/govready-q.git")
self.set_default(
'ADMINS',
[] if not self.config.get('ADMINS') else self.config.get('ADMINS'))
self.set_default(
'OKTA',
{} if not self.config.get('OKTA') else self.config.get('OKTA'))
self.set_default(
'OIDC',
{} if not self.config.get('OIDC') else self.config.get('OIDC'))
self.set_default('MOUNT_FOLDER', os.path.abspath("../../volumes"))
self.config['ALLOWED_HOSTS'] = ['app',
self.config['HOST_ADDRESS']] + getattr(
self.config, 'ALLOWED_HOSTS', [])
self.set_default('DEBUG', "false")
self.set_default('APP_DOCKER_PORT', "18000")
if self.check_if_valid_uri(self.config['HOST_ADDRESS']):
Prompt.error(
f"HOST_ADDRESS cannot be a valid URI. It must be the domain only. "
f"No protocol or path. {self.config['HOST_ADDRESS']} is invalid.",
close=True)
self.set_default('HEALTH_CHECK_GOVREADY_Q',
f"http://*****:*****@postgres:5432/govready_q")
self.set_default(
'DB_ENGINE',
self.config['DATABASE_CONNECTION_STRING'].split(':')[0])
docker_compose_file = "docker-compose.yaml"
if using_internal_db:
self.REQUIRED_PORTS.append(5432)
else:
docker_compose_file = 'docker-compose.external-db.yaml'
self.execute(
cmd=
f"docker-compose -f {docker_compose_file} down {self.FAIL_SUFFIX}")
self.REQUIRED_PORTS += [
int(self.config['HOST_PORT_HTTPS']),
int(self.config['APP_DOCKER_PORT'])
]
self.check_ports()
self.execute(cmd=f"docker-compose -f {docker_compose_file} up -d",
show_env=True)
def __docker_build_tmp_files_cleanup(self, base_path, file_path,
tmp_build):
if os.path.isdir(file_path):
dest = os.path.join(base_path,
f"{os.path.basename(file_path)}.zip")
else:
dest = os.path.join(base_path, os.path.basename(file_path))
os.remove(dest)
Prompt.notice(
f"Removed build artifact for image: {tmp_build['image']} - {dest}")
def __docker_build_tmp_files_copy(self, base_path, file_path, tmp_build,
config, key):
if os.path.isdir(file_path):
dest = make_archive(
os.path.join(base_path, os.path.basename(file_path)), 'zip',
file_path)
config[f"BUILD_FILE_{key}"] = f"{os.path.basename(file_path)}.zip"
else:
dest = os.path.join(base_path, os.path.basename(file_path))
copyfile(file_path, dest)
config[f"BUILD_FILE_{key}"] = os.path.basename(dest)
Prompt.notice(
f"Copied build file for image: {tmp_build['image']} - {dest}")
def check_ports(self):
Prompt.notice(
f"Checking if ports are available for deployment: {self.REQUIRED_PORTS}"
)
import socket
ports_in_use = []
for port in self.REQUIRED_PORTS:
with closing(socket.socket(socket.AF_INET,
socket.SOCK_STREAM)) as sock:
if sock.connect_ex(('127.0.0.1', port)) == 0:
ports_in_use.append(port)
if ports_in_use:
Prompt.error(
f"Cannot deploy. The following ports are in use: {ports_in_use}",
close=True)
def docker_tmp_file_handler(self,
config,
docker_tmp_build_files,
copy=True):
for tmp_build in docker_tmp_build_files:
for key in tmp_build['keys']:
file_path = config[key]
if file_path:
if not os.path.exists(file_path):
Prompt.error(
f"{os.path.abspath(file_path)} does not exist.",
close=True)
base_path = os.path.join(config['ROOT_DIR'], 'images',
tmp_build['image'])
if copy:
self.__docker_build_tmp_files_copy(
base_path, file_path, tmp_build, config, key)
else:
self.__docker_build_tmp_files_cleanup(
base_path, file_path, tmp_build)
def generate(self):
skeleton = {}
for row in self.validation_config_data:
skeleton[row['key']] = ""
self.run(skeleton)
with open("configuration.json", 'w') as f:
json.dump(skeleton, f, indent=4, sort_keys=True)
Prompt.warning(
f"Configuration created: {Colors.CYAN}configuration.json")
Prompt.warning(
f"Please set values in the configuration in line with your needs.")
Prompt.warning(
f"Once fully configured. Deploy by running: {Colors.CYAN}python run.py deploy --type {self.options['type']} --config configuration.json"
)
def on_complete(self):
logs = self.execute(cmd=f"docker-compose logs", display_stdout=False)
auto_admin = re.findall(
'Created administrator account \(username: (admin)\) with password: ([a-zA-Z0-9#?!@$%^&*-]+)',
logs)
print()
if auto_admin:
Prompt.warning(
f"Created Administrator Account - {Colors.CYAN}{auto_admin[0][0]} / {auto_admin[0][1]} - {Colors.FAIL} This is the only time you will see this message so make sure to write this down!"
)
Prompt.warning(
f"Logs & Container Artifacts can be found in: {Colors.CYAN}{self.config['MOUNT_FOLDER']}"
)
url = f"https://{self.config['HOST_ADDRESS']}"
if self.config['HOST_PORT_HTTPS']:
url += f":{self.config['HOST_PORT_HTTPS']}"
Prompt.warning(f"Access application via Browser: {Colors.CYAN}{url}")
if not self.FAIL_SUFFIX:
Prompt.warning(
f"WARNING: PERSIST_STACK is enabled. This may prevent configurations made since "
f"the last creation of the stack. Persisting a stack will cache the container and won't "
f"rebuild with your new configurations.")
def execute(self,
cmd,
env_dict,
display_stdout=True,
on_error_fn=None,
show_env=False,
display_stderr=True):
env = os.environ.copy()
normalized_dict = {}
for key, value in env_dict.items():
if isinstance(value, (list, dict)):
value = json.dumps(value)
if value is None:
value = ""
normalized_dict[key] = value
env.update(normalized_dict)
output = ""
Prompt.notice(f"Executing command: {Colors.WARNING}{cmd}")
if show_env:
Prompt.notice(
f"Environment Variables: {json.dumps(env_dict, indent=4, sort_keys=True)}"
)
args = dict(stdout=subprocess.PIPE, bufsize=0, env=env, shell=True)
if not display_stderr:
args.update(dict(stderr=subprocess.DEVNULL))
with subprocess.Popen(cmd, **args) as proc:
for line in proc.stdout:
formatted = line.rstrip().decode('utf-8', 'ignore')
output += formatted
if display_stdout:
print(formatted)
if proc.returncode != 0:
if on_error_fn:
on_error_fn()
Prompt.error(
f"[{cmd}] Failed [code:{proc.returncode}]- {proc.stderr}",
close=True)
return output
def get_deployment_type(deployment_type_list):
Prompt.question_banner("Deployment Choices:", True)
Prompt.banner(deployment_type_list, prevent_all_option=True)
return deployment_type_list[Prompt.get_response("Choice: ", deployment_type_list) - 1]
def run(options):
path = f"deployments.{options['type']}.{options['action']}"
validator_config = os.path.join(f"{os.path.sep}".join(path.split('.')[:-1]), 'config-validator.json')
importlib.import_module(path)
if options['action'] == 'init':
init_classes = Initialize.__subclasses__()
if not init_classes:
Prompt.error(f"Unable to find class inheriting `Initialize` in {path}", close=True)
if not os.path.exists(validator_config):
Prompt.error(
f"File does not exist: {validator_config}. Each deployment type must have this file to validate required values for deployment.",
close=True)
init = init_classes[0](validator_config, options)
init.generate()
return
Prompt.warning(f"Attempting to [{options['action']}] using the [{options['type']}] method")
if options['action'] == 'deploy':
if not options['config']:
Prompt.error(f"Deployments require a configuration json that satisfies: [{validator_config}]. Please run: {Colors.CYAN}python run.py init", close=True)
deployment_classes = Deployment.__subclasses__()
if not deployment_classes:
Prompt.error(f"Unable to find class inheriting `Deployment` in {path}", close=True)
if not os.path.exists(validator_config):
Prompt.error(
f"File does not exist: {validator_config}. Each deployment type must have this file to validate required values for deployment.",
close=True)
deployment = deployment_classes[0](options, validator_config)
deployment.validate()
os.chdir(f"deployments/{options['type']}")
try:
deployment.run()
finally:
deployment.cleanup()
deployment.on_complete()
Prompt.success(f"Deployment complete - using the [{options['type']}] method")
else:
deployment_classes = UnDeploy.__subclasses__()
if not deployment_classes:
Prompt.error(f"Unable to find class inheriting `UnDeploy` in {path}", close=True)
remove_deployment = deployment_classes[0](options)
os.chdir(f"deployments/{options['type']}")
try:
remove_deployment.run()
finally:
remove_deployment.cleanup()
remove_deployment.on_complete()
Prompt.success(f"Deployment removed - using the [{options['type']}] method")
finally:
remove_deployment.cleanup()
remove_deployment.on_complete()
Prompt.success(f"Deployment removed - using the [{options['type']}] method")
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Deploy/Undeploy GovReadyQ')
parser.add_argument('action', help='The action to take (init, deploy, undeploy)')
parser.add_argument('--config', help='Config file - used to deploy process', required=False)
parser.add_argument('--type', help="(Optional) Deployment type; It will prompt you if you don't include.",
required=False)
args, unknown = parser.parse_known_args()
args = vars(args)
args['extra'] = unknown
valid_actions = ['deploy', 'undeploy', 'init']
if args['action'] not in valid_actions:
Prompt.error(f"{args['action']} is not a valid choice. Choices: {valid_actions}", close=True)
deployment_types = sorted([y for y in [x[0].split(os.path.sep)[-1] for x in os.walk('deployments')][1:] if
not y.startswith('__')])
if not args['type']:
args['type'] = get_deployment_type(deployment_types)
if args['type'] not in deployment_types:
Prompt.error(f"{args['type']} is not a valid choice. Choices: {deployment_types}", close=True)
run(args)
def on_complete(self):
print()
Prompt.warning(f"If you're not using an external database and would like to wipe your DB, run: {Colors.CYAN}docker volume rm govready-q_postgres-data")
def on_complete(self):
Prompt.notice(
f"Next step: {Colors.CYAN}python run.py deploy --type docker-compose --config configuration.json"
)
def offline():
Prompt.error(
"Docker Engine is offline. Please start before continuing.",
close=True)