def deploy_app(cfg: Configuration):
test_env = cfg.get_property(Configuration.TEST_ENV_KEY)
builds_dir = cfg.get_property('kustomize.build.dir')
interpolated_file = cfg.get_property('kustomize.interpolated.file')
deploy_file = Path(builds_dir, f"{test_env}_latest", interpolated_file)
logger.info(f"{test_env}:Deploy app from {deploy_file}")
cmd = ['kubectl', 'apply', '-f', str(deploy_file)]
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logger.error(p.stderr)
def psql_options(cfg: Configuration, pg_database: str = 'postgres') -> List[str]:
pg_port = cfg.get_property('db.port')
pg_host = cfg.get_property('db.host')
pg_user = cfg.get_property('app.db.user')
pg_role = cfg.get_property('app.db.role')
pg_password = cfg.get_property('app.db.password')
os.putenv('PGPASSWORD', pg_password)
db_options = ['-d', pg_database]
pg_options = ['-h', pg_host, '-p', pg_port, '-U', pg_user]
return db_options + pg_options
def clean_db(cfg: Configuration):
test_env = cfg.get_property(Configuration.TEST_ENV_KEY)
logger.info(f"{test_env}: Drop and create all DBs")
# sql/postgresql/clean_dbs.sql
# default db is postgres
opt = psql_options(cfg)
psql_cmd(opt, sql_folder='sql/postgresql', sql_file='clean_dbs.sql')
def safe_net(cfg: Configuration):
""" ensure that correct (i.e. equal to test env) k8s context is used """
test_env = cfg.get_property(Configuration.TEST_ENV_KEY)
logger.info(f"{test_env}:Safe net")
# kube.context is set as property for given test env
kube_context = cfg.get_property("kube.context")
current_context_cmd = ['kubectl', 'config', 'current-context']
p = subprocess.run(current_context_cmd, stdout=subprocess.PIPE)
cc = p.stdout.decode('utf-8').strip()
typer.echo(f"current context:{cc}")
use_context_cmd = ['kubectl', 'config', 'use-context', kube_context]
subprocess.run(use_context_cmd)
p = subprocess.run(current_context_cmd, stdout=subprocess.PIPE)
cc = p.stdout.decode('utf-8').strip()
typer.echo(f"current context:{cc}")
# by convention k8s context == test env
assert cc == test_env
def delete_app(cfg: Configuration):
builds_dir = cfg.get_property('kustomize.build.dir')
interpolated_file = cfg.get_property('kustomize.interpolated.file')
deploy_folder = cfg.get_property(DEPLOY_FOLDER)
deploy_path = Path(builds_dir, deploy_folder)
deploy_file = Path(deploy_path, interpolated_file)
logger.info(f"Delete app from {deploy_file}")
if deploy_file.exists():
logger.info(f"Backup {deploy_file}")
shutil.copy(deploy_file, Path(deploy_path, f'{interpolated_file}.bak'))
cmd = ['kubectl', 'delete', '-f', str(deploy_file)]
p = subprocess.run(cmd, stdout=subprocess.STDOUT, stderr=subprocess.PIPE)
logger.error(p.stderr.decode('utf-8').strip())
time.sleep(30)
cmd = ['kubectl', '-n', 'product', 'get', 'pods']
p = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logger.info(p.stdout.decode('utf-8').strip())
typer.echo(p.stdout.decode('utf-8').strip())
def process_screenshots(cfg: Configuration, output_dir: Path, test: str = None):
scripts_dir = cfg.get_property('lr.vugen.scripts.git.dir')
test_dir = Path(scripts_dir, test)
logger.info(f"screenshots from {test}")
s_s_s = list_files(Path(scripts_dir, test), file_type='.png')
s_s_dir = Path(output_dir, 'screenshots')
os.makedirs(s_s_dir, exist_ok=True)
for s_s in s_s_s:
shutil.copy(Path(test_dir, s_s), s_s_dir)
def build_deployment(cfg: Configuration):
from cpt.kustomize import KubernetesManifest
# read properties for given test env
builds_dir = cfg.get_property('kustomize.build.dir')
git_dir = cfg.get_property('kustomize.git.dir')
generic_file = cfg.get_property('kustomize.generic.file')
interpolated_file = cfg.get_property('kustomize.interpolated.file')
test_env = cfg.get_property(Configuration.TEST_ENV_KEY)
deploy_folder = cfg.get_property(DEPLOY_FOLDER)
backup_folder(builds_dir, deploy_folder)
# for sake of right initial state
git_checkout(git_dir, file='kustomization.yaml')
km = KubernetesManifest(builds_dir, git_dir, deploy_folder,
test_env=test_env, generic_m=generic_file,
interpolated_m=interpolated_file)
km.switch_loaded_config_map()
kustomize_build(Path(builds_dir), Path(git_dir), deploy_folder, generic_file)
# revert changes
git_checkout(git_dir, 'kustomization.yaml')
km.load_vars()
km.load_manifest()
km.save_sizing()
def redeploy_paas(test_env: str = typer.Option(..., help="Test environment", envvar="TEST_ENV"),
step: str = None):
"""
Provides both individual and aggregated steps to redeploy Kustomize based deployment
example : python bin/pycpt/redeploy_paas.py --test-env paas_ci --step new_deploy
expects cwd = CPT_PATH when loading resources
"""
folder = f"{test_env}_latest"
cfg = Configuration(test_env=test_env)
# subfolder under builds_dir = cfg.get_property('kustomize.build.dir')
cfg.add_property(DEPLOY_FOLDER, folder)
safe_net(cfg)
match step:
case 'build':
build_deployment(cfg)
case 'deploy':
deploy_app(cfg)
case 'del_app':
delete_app(cfg)
case 'del_pv':
delete_p_v(cfg)
case 'clean_db':
clean_db(cfg)
case 'set_jdbc':
set_jdbc_string(cfg)
case 'restore_db':
restore_dump(cfg)
case 'new_deploy':
new_deploy(cfg)
case 'clean_deploy':
clean_deploy(cfg)
case 'build_deploy':
build_deploy(cfg)
case 'psql':
# default value of pg_database=postgres
opt = psql_options(cfg, pg_database=cfg.get_property('app.db.name'))
psql_cmd(opt, sql_file='mmm_db_metrics.sql')
case _:
logger.error(f"Unknown step {step}")
def run_fe_tests(cfg: Configuration, output_path: Path, backup_path: Path):
tests: List[str] = cfg.get_property('lr.vugen.scripts').split(',')
fe_app_url = cfg.get_property('fe.app.url')
scripts_dir = cfg.get_property('lr.vugen.scripts.git.dir')
for test in tests:
check_output(backup_path, output_path)
test_dir = Path(scripts_dir, test)
test_usr = Path(test_dir, test + '.usr').resolve()
cmd = ['mdrv.exe', '-usr', f"{test_usr}", '-app_url', fe_app_url]
cmd_open_wait(cmd)
log_file = Path(test_dir, 'output.txt')
if log_file.exists():
process_fe_log(log_file, output_dir=output_path, log_origin=test)
shutil.copy(log_file, output_path)
# copy screenshots
process_screenshots(cfg, output_dir=output_path, test=test)
else:
m = f"{log_file} does not exist"
logger.error(m)
typer.echo(m)
p = archive_folder(src_path=output_path, dest_path=backup_path, dest_base_file_name=test)
logger.info(f"{test} archived with return code {p.returncode}")
def child_pipeline(test_env: str = typer.Option(..., help="Test environment", envvar="TEST_ENV"),
step: str = typer.Option(...)):
global logger
global log_path
log_folder = Path(f"log/{test_env}").resolve()
logger, log_path = file_name_logger(folder=str(log_folder), file_name=file_name)
typer.echo(f"Logging to {log_path}")
cfg = Configuration(test_env=test_env)
# run_tests = cfg.get_property('run.fe.tests')
gen_data_dir = cfg.get_property("generated.data.rel.dir")
backup_dir = cfg.get_property("backup.data.rel.dir")
# full paths
gen_data_path = Path(gen_data_dir).resolve()
backup_path = Path(backup_dir).resolve()
match step:
case 'fe_tests':
run_fe_tests(cfg, output_path=gen_data_path, backup_path=backup_path)
case 'api_tests':
run_api_tests(cfg, backup_path=backup_path, log_base_dir=str(log_folder))
case _:
logger.error(f"Unknown step {step}")
def run_api_tests(cfg: Configuration, backup_path: Path, log_base_dir: str, test_suite: str = 'generated'):
test_env = cfg.get_property(Configuration.TEST_ENV_KEY)
os.makedirs(Path(log_base_dir), exist_ok=True)
typer.echo(f"api test logging to: {log_base_dir}: {test_suite}.log, api_test_{test_suite}.log")
test_options: List[str] = [f"-Dtest.environment={test_env}",
f"-Dlogger.base.dir={log_base_dir}/",
f"-Dlogger.file.name={test_suite}.log",
f"-PsuiteFile={cfg.get_property('test.suite.rel.dir')}/{test_suite}.xml",
f"-PexcludeTests=**/*DataGen*"]
cmd = ['gradlew.bat', 'test'] + test_options
f = open(Path(log_base_dir, f'api_test_{test_suite}.log'), "w")
p = subprocess.Popen(cmd, stdout=f)
ret = p.wait()
logger.info(f"{cmd} ended by {ret}")
test_results = cfg.get_property('test.results.rel.dir')
t_r_path = Path(test_results).resolve()
# TODO same as check_output - unify
if t_r_path.exists():
p = archive_folder(src_path=t_r_path, dest_path=backup_path,
dest_base_file_name='test_results')
logger.info(f"Test results {t_r_path} backup with return code {p.returncode}")
if p.returncode == 0:
logger.info(f"backup returned {p.returncode}. Remove {t_r_path}")
shutil.rmtree(t_r_path)
from cpt.configuration import Configuration
if __name__ == '__main__':
p = Configuration(test_env='paas_ci')
i = 0
for key in sorted(p.properties.keys()):
i += 1
print(f"{i}. {key}={p.properties[key]}")
def restore_dump(cfg: Configuration):
test_env = cfg.get_property(Configuration.TEST_ENV_KEY)
logger.info(f"{test_env}:Restore DB dump")
def set_jdbc_string(cfg: Configuration):
test_env = cfg.get_property(Configuration.TEST_ENV_KEY)
logger.info(f"{test_env}Set correct host name for connections")
from pathlib import Path
from cpt.configuration import Configuration
from cpt.kustomize import KubernetesManifest, kustomize_build, git_checkout
if __name__ == '__main__':
TEST_ENV = "paas_ci"
p = Configuration(test_env=TEST_ENV)
KUSTOMIZATION_YAML = 'kustomization.yaml'
GENERIC_FILE = p.get_property('kustomize.generic.file')
INTERPOLATED_FILE = p.get_property('kustomize.interpolated.file')
KUST_GIT_DIR = p.get_property('kustomize.git.dir')
# KUST_BUILDS_DIR = f"{os.getcwd()}\\kustomize_builds"
KUST_BUILDS_DIR = p.get_property('kustomize.build.dir')
FOLDER = f"{TEST_ENV}_latest"
km = KubernetesManifest(KUST_BUILDS_DIR,
KUST_GIT_DIR,
FOLDER,
test_env=TEST_ENV,
generic_m=GENERIC_FILE,
interpolated_m=INTERPOLATED_FILE)
km.switch_loaded_config_map()
kustomize_build(Path(KUST_BUILDS_DIR), Path(KUST_GIT_DIR), FOLDER,
GENERIC_FILE)
git_checkout(Path(KUST_GIT_DIR), KUSTOMIZATION_YAML)
km.load_vars()
km.load_manifest()
km.save_sizing()
parser.add_argument(
'--subfolder',
type=str,
required=True,
help='env specific subfolder in kustomize build directory')
return parser
if __name__ == '__main__':
args = setup_arg_parser().parse_args()
test_env = os.getenv("TEST_ENV")
subfolder_args = args.subfolder
# load properties for given test env
# cwd = CPT_HOME
p = Configuration(test_env=test_env)
p.set_raw_properties()
p.set_properties()
# get required properties
builds_dir = p.get_property('kustomize.build.dir')
git_dir = p.get_property('kustomize.git.dir')
generic_file = p.get_property('kustomize.generic.file')
interpolated_file = p.get_property('kustomize.interpolated.file')
km = KubernetesManifest(builds_dir,
git_dir,
subfolder_args,
test_env=test_env,
generic_m=generic_file,
interpolated_m=interpolated_file)