def process_cluster_cli_params(config):
"""
Process cluster related cli parameters
Args:
config (pytest.config): Pytest config object
"""
cluster_path = get_cli_param(config, 'cluster_path')
# Importing here cause once the function is invoked we have already config
# loaded, so this is OK to import once you sure that config is loaded.
from oc.openshift_ops import OCP
if cluster_path:
OCP.set_kubeconfig(
os.path.join(cluster_path,
ocsci_config.RUN['kubeconfig_location']))
# TODO: determine better place for parent dir
cluster_dir_parent = "/tmp"
default_cluster_name = ocs.defaults.CLUSTER_NAME
cluster_name = get_cli_param(config, 'cluster_name')
if not cluster_name:
cluster_name = default_cluster_name
cid = random.randint(10000, 99999)
if not (cluster_name and cluster_path):
cluster_name = f"{cluster_name}-{cid}"
if not cluster_path:
cluster_path = os.path.join(cluster_dir_parent, cluster_name)
ocsci_config.ENV_DATA['cluster_name'] = cluster_name
ocsci_config.ENV_DATA['cluster_path'] = cluster_path
def run(**kwargs):
log.info("Running OCS basic installation")
test_data = kwargs.get('test_data')
cluster_path = test_data.get('cluster-path')
# Test cluster access and if exist just skip the deployment.
if cluster_path and OCP.set_kubeconfig(
os.path.join(cluster_path, default.KUBECONFIG_LOCATION)):
return TestStatus.SKIPPED
config = kwargs.get('config')
cluster_conf = kwargs.get('cluster_conf')
workers = masters = aws_region = None
if cluster_conf:
cluster_details = cluster_conf.get('aws', {}).get('cluster', {})
workers = cluster_details.get('workers')
masters = cluster_details.get('masters')
aws_region = cluster_details.get('region', default.AWS_REGION)
# Generate install-config from template
log.info("Generating install-config")
# TODO: determine better place to create cluster directories - (log dir?)
cluster_dir_parent = "/tmp"
cluster_name = test_data.get('cluster-name')
base_cluster_name = test_data.get('cluster-name', default.CLUSTER_NAME)
cid = random.randint(10000, 99999)
if not (cluster_name and cluster_path):
cluster_name = f"{base_cluster_name}-{cid}"
if not cluster_path:
cluster_path = os.path.join(cluster_dir_parent, cluster_name)
run_cmd(f"mkdir -p {cluster_path}")
pull_secret_path = os.path.join(templating.TOP_DIR, "data", "pull-secret")
with open(pull_secret_path, "r") as f:
pull_secret = f.readline()
data = {
"cluster_name": cluster_name,
"pull_secret": pull_secret,
}
if workers:
data.update({'worker_replicas': workers})
if masters:
data.update({'master_replicas': masters})
if aws_region:
data.update({'region': aws_region})
_templating = templating.Templating()
template = _templating.render_template("install-config.yaml.j2", data)
log.info(f"Install config: \n{template}")
install_config = os.path.join(cluster_path, "install-config.yaml")
with open(install_config, "w") as f:
f.write(template)
# Download installer
installer_filename = "openshift-install"
tarball = f"{installer_filename}.tar.gz"
if os.path.isfile(installer_filename):
log.info("Installer exists, skipping download")
else:
log.info("Downloading openshift installer")
ver = config.get('installer-version', default.INSTALLER_VERSION)
if platform.system() == "Darwin":
os_type = "mac"
elif platform.system() == "Linux":
os_type = "linux"
else:
raise UnsupportedOSType
url = (f"https://mirror.openshift.com/pub/openshift-v4/clients/ocp/"
f"{ver}/openshift-install-{os_type}-{ver}.tar.gz")
download_file(url, tarball)
run_cmd(f"tar xzvf {tarball}")
# Deploy cluster
log.info("Deploying cluster")
run_cmd(f"./openshift-install create cluster "
f"--dir {cluster_path} "
f"--log-level debug")
# Test cluster access
if not OCP.set_kubeconfig(
os.path.join(cluster_path, default.KUBECONFIG_LOCATION)):
return TestStatus.FAILED
# TODO: Create cluster object, add to test_data for other tests to utilize
# Determine worker pattern and create ebs volumes
with open(os.path.join(cluster_path, "terraform.tfvars")) as f:
tfvars = json.load(f)
cluster_id = tfvars['cluster_id']
worker_pattern = f'{cluster_id}-worker*'
log.info(f'Worker pattern: {worker_pattern}')
region_name = aws_region if aws_region else default.AWS_REGION
create_ebs_volumes(worker_pattern, region_name=region_name)
# Use Rook to install Ceph cluster
# retrieve rook config from cluster_conf
rook_data = {}
if cluster_conf:
rook_data = cluster_conf.get('rook', {})
# render templates and create resources
create_oc_resource('common.yaml', rook_data, cluster_path, _templating)
run_cmd('oc label namespace openshift-storage '
'"openshift.io/cluster-monitoring=true"')
run_cmd("oc policy add-role-to-user view "
"system:serviceaccount:openshift-monitoring:prometheus-k8s "
"-n openshift-storage")
create_oc_resource('operator-openshift.yaml', rook_data, cluster_path,
_templating)
wait_time = 5
log.info(f"Waiting {wait_time} seconds...")
time.sleep(wait_time)
run_cmd("oc wait --for condition=ready pod "
"-l app=rook-ceph-operator "
"-n openshift-storage "
"--timeout=120s")
run_cmd("oc wait --for condition=ready pod "
"-l app=rook-ceph-agent "
"-n openshift-storage "
"--timeout=120s")
run_cmd("oc wait --for condition=ready pod "
"-l app=rook-discover "
"-n openshift-storage "
"--timeout=120s")
create_oc_resource('cluster.yaml', rook_data, cluster_path, _templating)
create_oc_resource('toolbox.yaml', rook_data, cluster_path, _templating)
log.info(f"Waiting {wait_time} seconds...")
time.sleep(wait_time)
create_oc_resource('storage-manifest.yaml', rook_data, cluster_path,
_templating)
create_oc_resource("service-monitor.yaml", rook_data, cluster_path,
_templating)
create_oc_resource("prometheus-rules.yaml", rook_data, cluster_path,
_templating)
# Verify health of ceph cluster
# TODO: move destroy cluster logic to new CLI usage pattern?
log.info("Done creating rook resources, waiting for HEALTH_OK")
rc = ceph_health_check()
# Destroy cluster (if configured)
destroy_cmd = (f"./openshift-install destroy cluster "
f"--dir {cluster_path} "
f"--log-level debug")
if config.get("destroy-cluster"):
log.info("Destroying cluster")
run_cmd(destroy_cmd)
# TODO: destroy volumes created
os.remove(installer_filename)
os.remove(tarball)
else:
log.info(f"Cluster directory is located here: {cluster_path}")
log.info(
f"Skipping cluster destroy. "
f"To manually destroy the cluster execute the following cmd:\n"
f"{destroy_cmd}")
return rc
def run(**kwargs):
log.info("Running OCS basic installation")
test_data = kwargs.get('test_data')
cluster_path = test_data.get('cluster-path')
# Test cluster access and if exist just skip the deployment.
if cluster_path and OCP.set_kubeconfig(
os.path.join(cluster_path, default.KUBECONFIG_LOCATION)):
return TestStatus.SKIPPED
config = kwargs.get('config')
cluster_conf = kwargs.get('cluster_conf', {})
env_data = deepcopy(default.ENV_DATA)
custom_env_data = cluster_conf.get('env_data', {})
# Generate install-config from template
log.info("Generating install-config")
# TODO: determine better place to create cluster directories - (log dir?)
cluster_dir_parent = "/tmp"
cluster_name = test_data.get('cluster-name')
base_cluster_name = test_data.get('cluster-name', default.CLUSTER_NAME)
cid = random.randint(10000, 99999)
if not (cluster_name and cluster_path):
cluster_name = f"{base_cluster_name}-{cid}"
if not cluster_path:
cluster_path = os.path.join(cluster_dir_parent, cluster_name)
run_cmd(f"mkdir -p {cluster_path}")
pull_secret_path = os.path.join(templating.TOP_DIR, "data", "pull-secret")
with open(pull_secret_path, "r") as f:
pull_secret = f.readline()
custom_env_data.update({
'pull_secret': pull_secret,
'cluster_name': cluster_name,
})
if custom_env_data:
env_data.update(custom_env_data)
# TODO: check for supported platform and raise the exception if not
# supported. Currently we support just AWS.
_templating = templating.Templating()
template = _templating.render_template("install-config.yaml.j2", env_data)
log.info(f"Install config: \n{template}")
install_config = os.path.join(cluster_path, "install-config.yaml")
with open(install_config, "w") as f:
f.write(template)
# Download installer
version = config.get('installer-version', default.INSTALLER_VERSION)
installer = download_openshift_installer(version)
# Deploy cluster
log.info("Deploying cluster")
run_cmd(f"./{installer} create cluster "
f"--dir {cluster_path} "
f"--log-level debug")
# Test cluster access
if not OCP.set_kubeconfig(
os.path.join(cluster_path, default.KUBECONFIG_LOCATION)):
return TestStatus.FAILED
# TODO: Create cluster object, add to test_data for other tests to utilize
# Determine worker pattern and create ebs volumes
with open(os.path.join(cluster_path, "terraform.tfvars")) as f:
tfvars = json.load(f)
cluster_id = tfvars['cluster_id']
worker_pattern = f'{cluster_id}-worker*'
log.info(f'Worker pattern: {worker_pattern}')
create_ebs_volumes(worker_pattern, region_name=env_data['region'])
# render templates and create resources
create_oc_resource('common.yaml', cluster_path, _templating, env_data)
run_cmd(f'oc label namespace {env_data["cluster_namespace"]} '
f'"openshift.io/cluster-monitoring=true"')
run_cmd(f"oc policy add-role-to-user view "
f"system:serviceaccount:openshift-monitoring:prometheus-k8s "
f"-n {env_data['cluster_namespace']}")
create_oc_resource('operator-openshift.yaml', cluster_path, _templating,
env_data)
# Increased to 10 seconds as 5 is not enough
# TODO: do the sampler function and check if resource exist
wait_time = 10
log.info(f"Waiting {wait_time} seconds...")
time.sleep(wait_time)
run_cmd(f"oc wait --for condition=ready pod "
f"-l app=rook-ceph-operator "
f"-n {env_data['cluster_namespace']} "
f"--timeout=120s")
run_cmd(f"oc wait --for condition=ready pod "
f"-l app=rook-ceph-agent "
f"-n {env_data['cluster_namespace']} "
f"--timeout=120s")
run_cmd(f"oc wait --for condition=ready pod "
f"-l app=rook-discover "
f"-n {env_data['cluster_namespace']} "
f"--timeout=120s")
create_oc_resource('cluster.yaml', cluster_path, _templating, env_data)
create_oc_resource('toolbox.yaml', cluster_path, _templating, env_data)
log.info(f"Waiting {wait_time} seconds...")
time.sleep(wait_time)
create_oc_resource('storage-manifest.yaml', cluster_path, _templating,
env_data)
create_oc_resource("service-monitor.yaml", cluster_path, _templating,
env_data)
create_oc_resource("prometheus-rules.yaml", cluster_path, _templating,
env_data)
# Verify health of ceph cluster
# TODO: move destroy cluster logic to new CLI usage pattern?
log.info("Done creating rook resources, waiting for HEALTH_OK")
rc = ceph_health_check(namespace=env_data['cluster_namespace'])
return rc
def test_deployment(self):
log.info("Running OCS basic installation")
cluster_path = ENV_DATA['cluster_path']
# Test cluster access and if exist just skip the deployment.
if RUN['cli_params'].get('cluster_path') and OCP.set_kubeconfig(
os.path.join(cluster_path, RUN.get('kubeconfig_location'))
):
pytest.skip(
"The installation is skipped cause the cluster is running"
)
# Generate install-config from template
log.info("Generating install-config")
run_cmd(f"mkdir -p {cluster_path}")
pull_secret_path = os.path.join(
TOP_DIR,
"data",
"pull-secret"
)
# TODO: check for supported platform and raise the exception if not
# supported. Currently we support just AWS.
_templating = templating.Templating()
install_config_str = _templating.render_template(
"install-config.yaml.j2", ENV_DATA
)
# Parse the rendered YAML so that we can manipulate the object directly
install_config_obj = yaml.safe_load(install_config_str)
with open(pull_secret_path, "r") as f:
# Parse, then unparse, the JSON file.
# We do this for two reasons: to ensure it is well-formatted, and
# also to ensure it ends up as a single line.
install_config_obj['pullSecret'] = json.dumps(json.loads(f.read()))
install_config_str = yaml.safe_dump(install_config_obj)
log.info(f"Install config: \n{install_config_str}")
install_config = os.path.join(cluster_path, "install-config.yaml")
with open(install_config, "w") as f:
f.write(install_config_str)
# Download installer
installer = get_openshift_installer(
DEPLOYMENT['installer_version']
)
# Download client
get_openshift_client()
# Deploy cluster
log.info("Deploying cluster")
run_cmd(
f"{installer} create cluster "
f"--dir {cluster_path} "
f"--log-level debug"
)
# Test cluster access
if not OCP.set_kubeconfig(
os.path.join(cluster_path, RUN.get('kubeconfig_location'))
):
pytest.fail("Cluster is not available!")
# TODO: Create cluster object, add to ENV_DATA for other tests to
# utilize.
# Determine worker pattern and create ebs volumes
with open(os.path.join(cluster_path, "terraform.tfvars")) as f:
tfvars = json.load(f)
cluster_id = tfvars['cluster_id']
worker_pattern = f'{cluster_id}-worker*'
log.info(f'Worker pattern: {worker_pattern}')
create_ebs_volumes(worker_pattern, region_name=ENV_DATA['region'])
# render templates and create resources
create_oc_resource('common.yaml', cluster_path, _templating, ENV_DATA)
run_cmd(
f'oc label namespace {ENV_DATA["cluster_namespace"]} '
f'"openshift.io/cluster-monitoring=true"'
)
run_cmd(
f"oc policy add-role-to-user view "
f"system:serviceaccount:openshift-monitoring:prometheus-k8s "
f"-n {ENV_DATA['cluster_namespace']}"
)
apply_oc_resource(
'csi-nodeplugin-rbac_rbd.yaml',
cluster_path,
_templating,
ENV_DATA,
template_dir="ocs-deployment/csi/rbd/"
)
apply_oc_resource(
'csi-provisioner-rbac_rbd.yaml',
cluster_path,
_templating,
ENV_DATA,
template_dir="ocs-deployment/csi/rbd/"
)
apply_oc_resource(
'csi-nodeplugin-rbac_cephfs.yaml',
cluster_path,
_templating,
ENV_DATA,
template_dir="ocs-deployment/csi/cephfs/"
)
apply_oc_resource(
'csi-provisioner-rbac_cephfs.yaml',
cluster_path,
_templating,
ENV_DATA,
template_dir="ocs-deployment/csi/cephfs/"
)
# Increased to 15 seconds as 10 is not enough
# TODO: do the sampler function and check if resource exist
wait_time = 15
log.info(f"Waiting {wait_time} seconds...")
time.sleep(wait_time)
create_oc_resource(
'operator-openshift-with-csi.yaml', cluster_path, _templating, ENV_DATA
)
log.info(f"Waiting {wait_time} seconds...")
time.sleep(wait_time)
run_cmd(
f"oc wait --for condition=ready pod "
f"-l app=rook-ceph-operator "
f"-n {ENV_DATA['cluster_namespace']} "
f"--timeout=120s"
)
run_cmd(
f"oc wait --for condition=ready pod "
f"-l app=rook-ceph-agent "
f"-n {ENV_DATA['cluster_namespace']} "
f"--timeout=120s"
)
run_cmd(
f"oc wait --for condition=ready pod "
f"-l app=rook-discover "
f"-n {ENV_DATA['cluster_namespace']} "
f"--timeout=120s"
)
create_oc_resource('cluster.yaml', cluster_path, _templating, ENV_DATA)
create_oc_resource('toolbox.yaml', cluster_path, _templating, ENV_DATA)
log.info(f"Waiting {wait_time} seconds...")
time.sleep(wait_time)
create_oc_resource(
'storage-manifest.yaml', cluster_path, _templating, ENV_DATA
)
create_oc_resource(
"service-monitor.yaml", cluster_path, _templating, ENV_DATA
)
create_oc_resource(
"prometheus-rules.yaml", cluster_path, _templating, ENV_DATA
)
log.info(f"Waiting {wait_time} seconds...")
time.sleep(wait_time)
# Verify health of ceph cluster
# TODO: move destroy cluster logic to new CLI usage pattern?
log.info("Done creating rook resources, waiting for HEALTH_OK")
assert ceph_health_check(namespace=ENV_DATA['cluster_namespace'])
def run(args):
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
suite_file = args['--suite']
conf_file = args['--conf']
store = args.get('--store', False)
reuse = args.get('--reuse', None)
post_results = args.get('--post-results')
cleanup_name = args.get('--cleanup', None)
post_to_report_portal = args.get('--report-portal', False)
cluster_name = args.get('--cluster-name')
cluster_path = args.get('--cluster-path')
send_email = not args.get('--no-email', False)
if cleanup_name:
pass # TODO: cleanup cluster and skip test execution
# Check if we are running on windows and bail out sooner
if platform.system() == "Windows":
log.error("Windows OS is not supported by Openshift Installer")
return ReturnCode.UNSUPPORTED_WINDOWS_RUN
if suite_file:
suites_path = os.path.abspath(suite_file)
with open(suites_path, 'r') as suite_stream:
suite = yaml.safe_load(suite_stream)
cluster_conf = dict()
if conf_file:
with open(conf_file) as f:
cluster_conf = yaml.safe_load(f)
# TODO: determine ci-message structure and necessity for OCS testing
if os.environ.get("TOOL") is not None:
pass
# TODO: determine ci-message structure and utilize for OCS if necessary
# otherwise remove logic
rp_service = None
suite_name = str(os.path.basename(suite_file).split(".")[0])
if post_to_report_portal:
log.info("Creating report portal session")
rp_service = create_report_portal_session()
# TODO: add appropriate values to report portal test description
launch_desc = textwrap.dedent(f"""
invoked-by: {getuser()}
""")
rp_service.start_launch(name=suite_name,
start_time=timestamp(),
description=launch_desc)
if reuse:
pass
# TODO: build cluster object and skip install
# potentially with additional kwargs read in by test(s)
if store:
pass
# TODO: store cluster data for non-aws installations,
# standardize location for cluster info
sys.path.append(os.path.abspath('tests'))
tests = suite.get('tests')
tcs = []
jenkins_rc = TestStatus.FAILED
test_data = dict()
if cluster_name:
test_data['cluster-name'] = cluster_name
if cluster_path:
OCP.set_kubeconfig(
os.path.join(cluster_path, default.KUBECONFIG_LOCATION))
test_data['cluster-path'] = cluster_path
for test in tests:
config = test.get('test').get('config', {})
test_kwargs = dict()
test_kwargs.update({'config': config})
test_kwargs.update({'test_data': test_data})
if cluster_conf:
test_kwargs.update({'cluster_conf': cluster_conf})
tc = TestCase(test.get('test'), suite_name, run_dir, test_kwargs,
rp_service)
tc.execute()
tcs.append(tc)
if tc.abort_on_fail and tc.status == TestStatus.FAILED:
log.info("Aborting on test failure")
break
if all([tc.status == TestStatus.PASSED for tc in tcs]):
jenkins_rc = TestStatus.PASSED
close_and_remove_filehandlers()
if post_to_report_portal:
rp_service.finish_launch(end_time=timestamp())
rp_service.terminate()
# TODO: need a new directory for ocs test logs?
# Once fixed, we can reuse code commented out below.
# url_base = "http://magna002.ceph.redhat.com/cephci-jenkins"
# run_dir_name = run_dir.split('/')[-1]
# print(f"\nAll test logs located here: {url_base}/{run_dir_name}")
print(f"\nAll test logs located here: {run_dir}/")
print_results(tcs)
send_to_qe = post_results or post_to_report_portal
if send_email:
email_results(tcs, run_id, send_to_qe)
return jenkins_rc