def cleanup(self): """ Clean up """ for project in self.projects: log.info(f"Delete Jenkins project: {project}") ocp_obj = OCP(namespace=project) ocp_obj.delete_project(project) log.info("Delete Jenkins Template") ocp_obj = OCP() cmd = "delete template.template.openshift.io/jenkins-persistent-ocs -n openshift" ocp_obj.exec_oc_cmd(command=cmd, out_yaml_format=False) # Wait for the resources to delete # https://github.com/red-hat-storage/ocs-ci/issues/2417 time.sleep(120)
class ElasticSearch(object): """ ElasticSearch Environment """ def __init__(self): """ Initializer function """ log.info("Initializing the Elastic-Search environment object") self.namespace = "elastic-system" self.eck_file = "ocs_ci/templates/app-pods/eck.1.3.1-all-in-one.yaml" self.dumper_file = "ocs_ci/templates/app-pods/esclient.yaml" self.pvc = "ocs_ci/templates/app-pods/es-pvc.yaml" self.crd = "ocs_ci/templates/app-pods/esq.yaml" # Creating some different types of OCP objects self.ocp = OCP(kind="pod", resource_name="elastic-operator-0", namespace=self.namespace) self.ns_obj = OCP(kind="namespace", namespace=self.namespace) self.es = OCP(resource_name="quickstart-es-http", namespace=self.namespace) self.elasticsearch = OCP(namespace=self.namespace, kind="elasticsearch") self.password = OCP( kind="secret", resource_name="quickstart-es-elastic-user", namespace=self.namespace, ) # Deploy the ECK all-in-one.yaml file self._deploy_eck() # Deploy the Elastic-Search server self._deploy_es() # Verify that ES is Up & Running timeout = 600 while timeout > 0: if self.get_health(): log.info("The ElasticSearch server is ready !") break else: log.warning("The ElasticSearch server is not ready yet") log.info("going to sleep for 30 sec. before next check") time.sleep(30) timeout -= 30 self._deploy_data_dumper_client() # Connect to the server self.con = self._es_connect() def _deploy_eck(self): """ Deploying the ECK environment for the Elasticsearch, and make sure it is in Running mode """ log.info("Deploying the ECK environment for the ES cluster") self.ocp.apply(self.eck_file) for es_pod in TimeoutSampler(300, 10, get_pod_name_by_pattern, "elastic-operator", self.namespace): try: if es_pod[0] is not None: self.eckpod = es_pod[0] log.info(f"The ECK pod {self.eckpod} is ready !") break except IndexError: log.info("ECK operator pod not ready yet") def _deploy_data_dumper_client(self): """ Deploying elastic search client pod with utility which dump all the data from the server to .tgz file """ log.info("Deploying the es client for dumping all data") self.ocp.apply(self.dumper_file) for dmp_pod in TimeoutSampler(300, 10, get_pod_name_by_pattern, "es-dumper", self.namespace): try: if dmp_pod[0] is not None: self.dump_pod = dmp_pod[0] log.info( f"The dumper client pod {self.dump_pod} is ready !") break except IndexError: log.info("Dumper pod not ready yet") def get_ip(self): """ This function return the IP address of the Elasticsearch cluster. this IP is to use inside the OCP cluster Return str : String that represent the Ip Address. """ return self.es.get()["spec"]["clusterIP"] def get_port(self): """ This function return the port of the Elasticsearch cluster. Return str : String that represent the port. """ return self.es.get()["spec"]["ports"][0]["port"] def _deploy_es(self): log.info("Deploy the PVC for the ElasticSearch cluster") self.ocp.apply(self.pvc) log.info("Deploy the ElasticSearch cluster") self.ocp.apply(self.crd) for es_pod in TimeoutSampler(300, 20, get_pod_name_by_pattern, "quickstart-es-default", self.namespace): try: if es_pod[0] is not None: self.espod = es_pod[0] log.info(f"The ElasticSearch pod {self.espod} Started") break except IndexError: log.info("elasticsearch pod not ready yet") es_pod = OCP(kind="pod", namespace=self.namespace) log.info("Waiting for ElasticSearch to Run") assert es_pod.wait_for_resource( condition=constants.STATUS_RUNNING, resource_name=self.espod, sleep=30, timeout=600, ) log.info("Elastic Search is ready !!!") def get_health(self): """ This method return the health status of the Elasticsearch. Returns: bool : True if the status is green (OK) otherwise - False """ return self.elasticsearch.get( )["items"][0]["status"]["health"] == "green" def get_password(self): """ This method return the password used to connect the Elasticsearch. Returns: str : The password as text """ return base64.b64decode( self.password.get()["data"]["elastic"]).decode("utf-8") def cleanup(self): """ Cleanup the environment from all Elasticsearch components, and from the port forwarding process. """ log.info("Teardown the Elasticsearch environment") log.info("Deleting all resources") log.info("Deleting the dumper client pod") self.ocp.delete(yaml_file=self.dumper_file) log.info("Deleting the es resource") self.ocp.delete(yaml_file=self.crd) log.info("Deleting the es project") self.ns_obj.delete_project(project_name=self.namespace) self.ns_obj.wait_for_delete(resource_name=self.namespace, timeout=180) def _es_connect(self): """ Create a connection to the local ES Returns: Elasticsearch: elasticsearch connection object Raise: ConnectionError: if can not connect to the server """ try: es = Elasticsearch([{ "host": self.get_ip(), "port": self.get_port() }]) except esexp.ConnectionError: log.error("Can not connect to ES server in the LocalServer") raise return es def get_indices(self): """ Getting list of all indices in the ES server - all created by the test, the installation of the ES was without any indexes pre-installed. Returns: list : list of all indices defined in the ES server """ results = [] log.info("Getting all indices") for ind in self.con.indices.get_alias("*"): results.append(ind) return results def _copy(self, es): """ Copy All data from the internal ES server to the main ES. **This is deprecated function** , use the dump function, and load the data from the files for the main ES server Args: es (obj): elasticsearch object which connected to the main ES """ query = {"size": 1000, "query": {"match_all": {}}} for ind in self.get_indices(): log.info(f"Reading {ind} from internal ES server") try: result = self.con.search(index=ind, body=query) except esexp.NotFoundError: log.warning(f"{ind} Not found in the Internal ES.") continue log.debug(f"The results from internal ES for {ind} are :{result}") log.info(f"Writing {ind} into main ES server") for doc in result["hits"]["hits"]: log.debug(f"Going to write : {doc}") es.index(index=ind, doc_type="_doc", body=doc["_source"]) def dumping_all_data(self, target_path): """ Dump All data from the internal ES server to .tgz file. Args: target_path (str): the path where the results file will be copy into Return: bool: True if the dump operation succeed and return the results data to the host otherwise False """ log.info("dumping data from ES server to .tgz file") rsh_cmd = f"rsh {self.dump_pod} /elasticsearch-dump/esdumper.py --ip {self.get_ip()} --port {self.get_port()}" result = self.ocp.exec_oc_cmd(rsh_cmd, out_yaml_format=False, timeout=1200) if "ES dump is done." not in result: log.error("There is no data in the Elasticsearch server") return False else: src_file = result.split()[-1] log.info(f"Copy {src_file} from the client pod") cp_command = f"cp {self.dump_pod}:{src_file} {target_path}/FullResults.tgz" result = self.ocp.exec_oc_cmd(cp_command, timeout=120) log.info(f"The output from the POD is {result}") log.info("Extracting the FullResults.tgz file") kwargs = {"cwd": target_path} results = run_command(f"tar zxvf {target_path}/FullResults.tgz", **kwargs) log.debug(f"The untar results is {results}") if "Error in command" in results: log.warning("Can not untar the dumped file") return False return True
class Cosbench(object): """ Cosbench S3 benchmark tool """ def __init__(self): """ Initializer function """ self.ns_obj = OCP(kind="namespace") self.namespace = constants.COSBENCH_PROJECT self.configmap_obj = OCP(namespace=self.namespace, kind=constants.CONFIGMAP) self.ocp_obj = OCP(namespace=self.namespace) self.cosbench_config = None self.cosbench_pod = None self.cosbench_dir = mkdtemp(prefix="cosbench-tool-") self.xml_file = "" self.workload_id = "" self.init_container = 1 self.range_selector = "r" self.init_object = 1 mcg_obj = MCG() self.access_key_id = mcg_obj.access_key_id self.access_key = mcg_obj.access_key self.endpoint = ( "http://" + mcg_obj.s3_internal_endpoint.split("/")[2].split(":")[0]) def setup_cosbench(self): """ Setups Cosbench namespace, configmap and pod """ # Create cosbench project self.ns_obj.new_project(project_name=self.namespace) # Create configmap config_data = templating.load_yaml(file=constants.COSBENCH_CONFIGMAP) cosbench_configmap_name = create_unique_resource_name( constants.COSBENCH, "configmap") config_data["metadata"]["name"] = cosbench_configmap_name config_data["metadata"]["namespace"] = self.namespace self.cosbench_config = OCS(**config_data) logger.info( f"Creating Cosbench configmap: {self.cosbench_config.name}") self.cosbench_config.create() self.configmap_obj.wait_for_resource( resource_name=self.cosbench_config.name, column="DATA", condition="4") # Create Cosbench pod cosbench_pod_data = templating.load_yaml(file=constants.COSBENCH_POD) cosbench_pod_data["spec"]["containers"][0]["envFrom"][0][ "configMapRef"]["name"] = self.cosbench_config.name cosbench_pod_name = create_unique_resource_name( constants.COSBENCH, "pod") cosbench_pod_data["metadata"]["name"] = cosbench_pod_name cosbench_pod_data["metadata"]["namespace"] = self.namespace self.cosbench_pod = OCS(**cosbench_pod_data) logger.info(f"Creating Cosbench pod: {self.cosbench_pod.name}") self.cosbench_pod.create() helpers.wait_for_resource_state(resource=self.cosbench_pod, state=constants.STATUS_RUNNING, timeout=300) def _apply_mcg_auth(self, xml_root): """ Applies MCG credentials Args: xml_root (Element): Root element of workload xml """ xml_root[0].set( "config", f"accesskey={self.access_key_id};secretkey={self.access_key};" f"endpoint={self.endpoint};path_style_access=true", ) def run_init_workload( self, prefix, containers, objects, start_container=None, start_object=None, size=64, size_unit="KB", sleep=15, timeout=300, validate=True, ): """ Creates specific containers and objects in bulk Args: prefix (str): Prefix of bucket name. containers (int): Number of containers/buckets to be created. objects (int): Number of objects to be created on each bucket. start_container (int): Start of containers. Default: 1. start_object (int): Start of objects. Default: 1. size (int): Size of each objects. size_unit (str): Object size unit (B/KB/MB/GB) sleep (int): Sleep in seconds. timeout (int): Timeout in seconds. validate (bool): Validates whether init and prepare is completed. Returns: Tuple[str, str]: Workload xml and its name """ init_template = """ <workload name="Fill" description="Init and prepare operation"> <storage type="s3" config="" /> <workflow> <workstage name="init-containers"> <work type="init" workers="1" config="" /> </workstage> <workstage name="prepare-objects"> <work type="prepare" workers="16" config="" /> </workstage> </workflow> </workload> """ xml_root, xml_tree = self._create_element_tree(template=init_template) workload_name = xml_root.get("name") self._apply_mcg_auth(xml_root) self.init_container = (start_container if start_container else self.init_container) self.init_object = start_object if start_object else self.init_object init_container_config = self.generate_container_stage_config( self.range_selector, self.init_container, containers, ) init_config = self.generate_stage_config( self.range_selector, self.init_container, containers, self.init_object, objects, ) for stage in xml_root.iter("work"): if stage.get("type") == "init": stage.set("config", f"cprefix={prefix};{init_container_config}") elif stage.get("type") == "prepare": stage.set( "config", f"cprefix={prefix};{init_config};sizes=c({str(size)}){size_unit}", ) self._create_tmp_xml(xml_tree=xml_tree, xml_file_prefix=workload_name) self.submit_workload(workload_path=self.xml_file) self.wait_for_workload(workload_id=self.workload_id, sleep=sleep, timeout=timeout) if validate: self.validate_workload(workload_id=self.workload_id, workload_name=workload_name) else: return self.workload_id, workload_name def run_cleanup_workload( self, prefix, containers, objects, start_container=None, start_object=None, sleep=15, timeout=300, validate=True, ): """ Deletes specific objects and containers in bulk. Args: prefix (str): Prefix of bucket name. containers (int): Number of containers/buckets to be created. objects (int): Number of objects to be created on each bucket. start_container (int): Start of containers. Default: 1. start_object (int): Start of objects. Default: 1. sleep (int): Sleep in seconds. timeout (int): Timeout in seconds. validate (bool): Validates whether cleanup and dispose is completed. Returns: Tuple[str, str]: Workload xml and its name """ cleanup_template = """ <workload name="Cleanup" description="Cleanup and Dispose"> <storage type="s3" config="" /> <workflow> <workstage name="cleanup-objects"> <work type="cleanup" workers="4" config="" /> </workstage> <workstage name="dispose-containers"> <work type="dispose" workers="1" config="" /> </workstage> </workflow> </workload> """ xml_root, xml_tree = self._create_element_tree( template=cleanup_template) workload_name = xml_root.get("name") self._apply_mcg_auth(xml_root) self.init_container = (start_container if start_container else self.init_container) self.init_object = start_object if start_object else self.init_object cleanuo_config = self.generate_stage_config( self.range_selector, self.init_container, containers, self.init_object, objects, ) for stage in xml_root.iter("work"): if stage.get("type") == "cleanup": stage.set( "config", f"cprefix={prefix};{cleanuo_config}", ) elif stage.get("type") == "dispose": stage.set("config", f"cprefix={prefix};{cleanuo_config}") self._create_tmp_xml(xml_tree=xml_tree, xml_file_prefix=workload_name) self.submit_workload(workload_path=self.xml_file) self.wait_for_workload(workload_id=self.workload_id, sleep=sleep, timeout=timeout) if validate: self.validate_workload(workload_id=self.workload_id, workload_name=workload_name) else: return self.workload_id, workload_name def run_main_workload( self, operation_type, prefix, containers, objects, workers=4, selector="s", start_container=None, start_object=None, size=64, size_unit="KB", sleep=15, timeout=300, extend_objects=None, validate=True, result=True, ): """ Creates and runs main Cosbench workload. Args: operation_type (dict): Cosbench operation and its ratio. Operation (str): Supported ops are read, write, list and delete. Ratio (int): Percentage of each operation. Should add up to 100. workers (int): Number of users to perform operations. containers (int): Number of containers/buckets to be created. objects (int): Number of objects to be created on each bucket. selector (str): The way object is accessed/selected. u=uniform, r=range, s=sequential. prefix (str): Prefix of bucket name. start_container (int): Start of containers. Default: 1. start_object (int): Start of objects. Default: 1. size (int): Size of each objects. size_unit (str): Object size unit (B/KB/MB/GB) sleep (int): Sleep in seconds timeout (int): Timeout in seconds validate (bool): Validates whether each stage is completed extend_objects (int): Extends the total number of objects to prevent overlap. Use only for Write and Delete operations. result (bool): Get performance results when running workload is completed. Returns: Tuple[str, str]: Workload xml and its name """ main_template = """ <workload name="workload_name" description="Main workload"> <storage type="s3" config="" /> <workflow> <workstage name="Main"> <work name="work_name" workers="4" division="object" runtime="60"> </work> </workstage> </workflow> </workload> """ xml_root, xml_tree = self._create_element_tree(template=main_template) workload_name = xml_root.get("name") self._apply_mcg_auth(xml_root) start_container = start_container if start_container else self.init_container start_object = start_object if start_object else self.init_object for stage in xml_root.iter("work"): stage.set("workers", f"{workers}") for operation, ratio in operation_type.items(): if operation == "write" or "delete": if extend_objects: start_object = objects + 1 stage_config = self.generate_stage_config( selector, start_container, containers, start_object, extend_objects, ) attributes = { "type": f"{operation}", "ratio": f"{ratio}", "config": f"cprefix={prefix};{stage_config};sizes=c({str(size)}){size_unit}", } ElementTree.SubElement(stage, "operation", attributes) else: stage_config = self.generate_stage_config( selector, start_container, containers, start_object, objects, ) attributes = { "type": f"{operation}", "ratio": f"{ratio}", "config": f"cprefix={prefix};{stage_config};sizes=c({str(size)}){size_unit}", } ElementTree.SubElement(stage, "operation", attributes) else: stage_config = self.generate_stage_config( selector, start_container, containers, start_object, objects, ) attributes = { "type": f"{operation}", "ratio": f"{ratio}", "config": f"cprefix={prefix};{stage_config}", } ElementTree.SubElement(stage, "operation", attributes) self._create_tmp_xml(xml_tree=xml_tree, xml_file_prefix=workload_name) self.submit_workload(workload_path=self.xml_file) self.wait_for_workload(workload_id=self.workload_id, sleep=sleep, timeout=timeout) if validate: self.validate_workload(workload_id=self.workload_id, workload_name=workload_name) else: return self.workload_id, workload_name if result: throughput, bandwidth = self.get_performance_result( workload_id=self.workload_id, workload_name=workload_name, size=size, ) return throughput, bandwidth else: return self.workload_id, workload_name @staticmethod def generate_stage_config(selector, start_container, end_container, start_objects, end_object): """ Generates config which is used in stage creation Args: selector (str): The way object is accessed/selected. u=uniform, r=range, s=sequential. start_container (int): Start of containers end_container (int): End of containers start_objects (int): Start of objects end_object (int): End of objects Returns: (str): Container and object configuration """ xml_config = ( f"containers={selector}({str(start_container)},{str(end_container)});" f"objects={selector}({str(start_objects)},{str(end_object)})") return xml_config @staticmethod def generate_container_stage_config(selector, start_container, end_container): """ Generates container config which creates buckets in bulk Args: selector (str): The way object is accessed/selected. u=uniform, r=range, s=sequential. start_container (int): Start of containers end_container (int): End of containers Returns: (str): Container and object configuration """ container_config = ( f"containers={selector}({str(start_container)},{str(end_container)});" ) return container_config def _create_tmp_xml(self, xml_tree, xml_file_prefix): """ Creates a xml file and writes the workload Args: xml_file_prefix (str): Prefix of xml file xml_tree (Element): Element tree """ self.xml_file = NamedTemporaryFile( dir=self.cosbench_dir, prefix=f"{xml_file_prefix}", suffix=".xml", delete=False, ).name logger.info(self.xml_file) xml_tree.write(self.xml_file) @staticmethod def _create_element_tree(template): """ Creates element tree and root element of xml Args: template (str): Template of Cosbench workload Returns: Tuple[Element, ElementTree]: Root element and element tree of xml """ xml_root = ElementTree.fromstring(text=template) xml_tree = ElementTree.ElementTree(element=xml_root) return xml_root, xml_tree def _copy_workload(self, workload_path): """ Copies workload xml to Cosbench pod Args: workload_path (str): Absolute path of xml to copy """ self.ocp_obj.exec_oc_cmd( command=f"cp {workload_path} {self.cosbench_pod.name}:/cos", out_yaml_format=False, timeout=180, ) def submit_workload(self, workload_path): """ Submits Cosbench xml to initiate workload Args: workload_path (str): Absolute path of xml to submit """ self._copy_workload(workload_path=workload_path) workload = os.path.split(workload_path)[1] self._cosbench_cli(workload) @retry(AttributeError, tries=15, delay=5, backoff=1) def _cosbench_cli(self, workload): """ Runs Cosbench cli to initiate workload Args: workload (str): Workload file """ submit_key = "Accepted with ID" cobench_pod_obj = get_pod_obj(name=self.cosbench_pod.name, namespace=self.namespace) submit = cobench_pod_obj.exec_cmd_on_pod( command=f"/cos/cli.sh submit /cos/{workload}", out_yaml_format=True, timeout=180, ) if submit_key in submit.keys(): self.workload_id = submit[submit_key] else: assert f"Failed to submit the workload, ID not found. stdout: {submit}" def wait_for_workload(self, workload_id, sleep=1, timeout=60): """ Waits for the cosbench workload to complete Args: workload_id (str): ID of cosbench workload sleep: sleep in seconds timeout: timeout in seconds to check if mirroring Returns: bool: Whether cosbench workload processed successfully """ logger.info(f"Waiting for workload {workload_id} to be processed") pattern = f"sucessfully processed workload {workload_id}" try: for ret in TimeoutSampler( timeout=timeout, sleep=sleep, func=get_pod_logs, pod_name=self.cosbench_pod.name, namespace=self.namespace, ): if re.search(pattern=pattern, string=ret): break logger.info( f"Verified: Workload {workload_id} processed successfully") return True except TimeoutExpiredError: logger.error( f"Workload {workload_id} did not complete. Dumping cosbench pod log" ) # Log cosbench pod for debugging purpose cosbench_log = get_pod_logs(pod_name=self.cosbench_pod.name, namespace=self.namespace) logger.debug(cosbench_log) return False def validate_workload(self, workload_id, workload_name): """ Validates each stage of cosbench workload Args: workload_id (str): ID of cosbench workload workload_name (str): Name of the workload Raises: UnexpectedBehaviour: When workload csv is incorrect/malformed. """ workload_csv = self.get_result_csv(workload_id=workload_id, workload_name=workload_name) with open(workload_csv, "r") as file: reader = csv.reader(file) header = next(reader) if header is not None: # Iterate over each row after the header logger.info( f"Verifying whether each stage of workload {workload_id} completed" ) for row in reader: if row[16] == "completed": logger.info(f"Stage {row[0]} completed successfully") else: assert ( f"Failed: Stage {row[0]} did not complete. Status {row[16]}" ) else: raise UnexpectedBehaviour( f"Workload csv is incorrect/malformed. Dumping csv {reader}" ) def get_result_csv(self, workload_id, workload_name): """ Gets cosbench workload result csv Args: workload_id (str): ID of cosbench workload workload_name (str): Name of the workload Returns: str: Absolute path of the result csv """ archive_file = f"{workload_id}-{workload_name}" cmd = ( f"cp {self.cosbench_pod.name}:/cos/archive/{archive_file}/{archive_file}.csv " f"{self.cosbench_dir}/{archive_file}.csv ") self.ocp_obj.exec_oc_cmd( command=cmd, out_yaml_format=False, timeout=300, ) return f"{self.cosbench_dir}/{archive_file}.csv" def cleanup(self): """ Cosbench cleanup """ switch_to_project(constants.COSBENCH_PROJECT) logger.info("Deleting Cosbench pod, configmap and namespace") self.cosbench_pod.delete() self.cosbench_config.delete() self.ns_obj.delete_project(self.namespace) self.ns_obj.wait_for_delete(resource_name=self.namespace, timeout=90) def get_performance_result(self, workload_name, workload_id, size): workload_file = self.get_result_csv(workload_id=workload_id, workload_name=workload_name) throughput_data = {} bandwidth_data = {} with open(workload_file, "r") as file: reader = csv.reader(file) header = next(reader) if header is not None: for row in reader: throughput_data[row[1]] = row[13] bandwidth_data[row[1]] = row[14] else: raise UnexpectedBehaviour( f"Workload csv is incorrect/malformed. Dumping csv {reader}" ) # Store throughput data on csv file log_path = f"{self.cosbench_dir}" with open(f"{log_path}/{workload_name}-{size}-throughput.csv", "a") as fd: csv_obj = csv.writer(fd) for k, v in throughput_data.items(): csv_obj.writerow([k, v]) logger.info( f"Throughput data present in {log_path}/{workload_name}-{size}-throughput.csv" ) # Store bandwidth data on csv file with open(f"{log_path}/{workload_name}-{size}-bandwidth.csv", "a") as fd: csv_obj = csv.writer(fd) for k, v in bandwidth_data.items(): csv_obj.writerow([k, v]) logger.info( f"Bandwidth data present in {log_path}/{workload_name}-{size}-bandwidth.csv" ) return throughput_data, bandwidth_data def cosbench_full(self): """ Run full Cosbench workload """ bucket_prefix = "bucket-" buckets = 10 objects = 1000 # Operations to perform and its ratio(%) operations = {"read": 50, "write": 50} # Deployment of cosbench self.setup_cosbench() # Create initial containers and objects self.run_init_workload(prefix=bucket_prefix, containers=buckets, objects=objects, validate=True) # Start measuring time start_time = datetime.now() # Run main workload self.run_main_workload( operation_type=operations, prefix=bucket_prefix, containers=buckets, objects=objects, validate=True, timeout=10800, ) # Calculate the total run time of Cosbench workload end_time = datetime.now() diff_time = end_time - start_time logger.info(f"Cosbench workload completed after {diff_time}") # Dispose containers and objects self.run_cleanup_workload(prefix=bucket_prefix, containers=buckets, objects=objects, validate=True)
class CouchBase(PillowFight): """ CouchBase workload operation """ def __init__(self, **kwargs): """ Initializer function """ super().__init__(**kwargs) self.args = kwargs self.pod_obj = OCP(kind="pod") self.ns_obj = OCP(kind="namespace") self.couchbase_pod = OCP(kind="pod") self.create_namespace(namespace=constants.COUCHBASE_OPERATOR) self.cb_create_cb_secret = False self.cb_create_cb_cluster = False self.cb_create_bucket = False def create_namespace(self, namespace): """ create namespace for couchbase Args: namespace (str): Namespace for deploying couchbase pods """ try: self.ns_obj.new_project(namespace) except CommandFailed as ef: log.info("Already present") if f'project.project.openshift.io "{namespace}" already exists' not in str( ef): raise ef def couchbase_operatorgroup(self): """ Creates an operator group for Couchbase """ operatorgroup_yaml = templating.load_yaml( constants.COUCHBASE_OPERATOR_GROUP_YAML) self.operatorgroup_yaml = OCS(**operatorgroup_yaml) self.operatorgroup_yaml.create() def couchbase_subscription(self): """ Creates subscription for Couchbase operator """ # Create an operator group for Couchbase log.info("Creating operator group for couchbase") self.couchbase_operatorgroup() subscription_yaml = templating.load_yaml( constants.COUCHBASE_OPERATOR_SUBSCRIPTION_YAML) self.subscription_yaml = OCS(**subscription_yaml) self.subscription_yaml.create() # Wait for the CSV to reach succeeded state cb_csv = self.get_couchbase_csv() cb_csv_obj = CSV(resource_name=cb_csv, namespace=constants.COUCHBASE_OPERATOR) cb_csv_obj.wait_for_phase("Succeeded", timeout=720) def get_couchbase_csv(self): """ " Get the Couchbase CSV object Returns: CSV: Couchbase CSV object Raises: CSVNotFound: In case no CSV found. """ cb_package_manifest = PackageManifest( resource_name="couchbase-enterprise-certified") cb_enter_csv = cb_package_manifest.get_current_csv( channel="stable", csv_pattern=constants.COUCHBASE_CSV_PREFIX) return cb_enter_csv def create_cb_secrets(self): """ " Create secrets for running Couchbase workers """ cb_secrets = templating.load_yaml(constants.COUCHBASE_WORKER_SECRET) self.cb_secrets = OCS(**cb_secrets) self.cb_secrets.create() log.info("Successfully created secrets for Couchbase") self.cb_create_cb_secret = True def create_cb_cluster(self, replicas=1, sc_name=None): """ Deploy a Couchbase server using Couchbase operator Once the couchbase operator is running, we need to wait for the worker pods to be up. Once the Couchbase worker pods are up, pillowfight task is started. After the pillowfight task has finished, the log is collected and analyzed. Raises: Exception: If pillowfight results indicate that a minimum performance level is not reached (1 second response time, less than 1000 ops per second) """ log.info("Creating Couchbase worker pods...") cb_example = templating.load_yaml(constants.COUCHBASE_WORKER_EXAMPLE) if storagecluster_independent_check(): cb_example["spec"]["volumeClaimTemplates"][0]["spec"][ "storageClassName"] = constants.DEFAULT_EXTERNAL_MODE_STORAGECLASS_RBD cb_example["spec"]["servers"][0]["size"] = replicas if sc_name: cb_example["spec"]["volumeClaimTemplates"][0]["spec"][ "storageClassName"] = sc_name self.cb_example = OCS(**cb_example) self.cb_example.create() self.cb_create_cb_cluster = True # Wait for the Couchbase workers to be running. log.info("Waiting for the Couchbase pods to be Running") self.pod_obj.wait_for_resource( condition="Running", selector="app=couchbase", resource_count=replicas, timeout=900, ) log.info( f"Expected number: {replicas} of couchbase workers reached running state" ) def create_data_buckets(self): """ Create data buckets """ cb_bucket = templating.load_yaml(constants.COUCHBASE_DATA_BUCKET) self.cb_bucket = OCS(**cb_bucket) self.cb_bucket.create() log.info("Successfully created data buckets") self.cb_create_bucket = True def run_workload(self, replicas, num_items=None, num_threads=None, run_in_bg=False): """ Running workload with pillow fight operator Args: replicas (int): Number of pods num_items (int): Number of items to be loaded to the cluster num_threads (int): Number of threads run_in_bg (bool) : Optional run IOs in background """ self.result = None log.info("Running IOs using Pillow-fight") if run_in_bg: executor = ThreadPoolExecutor(1) self.result = executor.submit( PillowFight.run_pillowfights, self, replicas=replicas, num_items=num_items, num_threads=num_threads, ) return self.result PillowFight.run_pillowfights(self, replicas=replicas, num_items=num_items, num_threads=num_threads) def analyze_run(self, skip_analyze=False): """ Analyzing the workload run logs Args: skip_analyze (bool): Option to skip logs analysis """ if not skip_analyze: log.info("Analyzing workload run logs..") PillowFight.analyze_all(self) def respin_couchbase_app_pod(self): """ Respin the couchbase app pod Returns: pod status """ app_pod_list = get_pod_name_by_pattern("cb-example", constants.COUCHBASE_OPERATOR) app_pod = app_pod_list[random.randint(0, len(app_pod_list) - 1)] log.info(f"respin pod {app_pod}") app_pod_obj = get_pod_obj(app_pod, namespace=constants.COUCHBASE_OPERATOR) app_pod_obj.delete(wait=True, force=False) wait_for_resource_state(resource=app_pod_obj, state=constants.STATUS_RUNNING, timeout=300) def get_couchbase_nodes(self): """ Get nodes that contain a couchbase app pod Returns: list: List of nodes """ app_pods_list = get_pod_name_by_pattern("cb-example", constants.COUCHBASE_OPERATOR) app_pod_objs = list() for pod in app_pods_list: app_pod_objs.append( get_pod_obj(pod, namespace=constants.COUCHBASE_OPERATOR)) log.info("Create a list of nodes that contain a couchbase app pod") nodes_set = set() for pod in app_pod_objs: log.info(f"pod {pod.name} located on " f"node {pod.get().get('spec').get('nodeName')}") nodes_set.add(pod.get().get("spec").get("nodeName")) return list(nodes_set) def teardown(self): """ Cleaning up the resources created during Couchbase deployment """ if self.cb_create_cb_secret: self.cb_secrets.delete() if self.cb_create_cb_cluster: self.cb_example.delete() if self.cb_create_bucket: self.cb_bucket.delete() self.subscription_yaml.delete() switch_to_project("default") self.ns_obj.delete_project(constants.COUCHBASE_OPERATOR) self.ns_obj.wait_for_delete(resource_name=constants.COUCHBASE_OPERATOR, timeout=90) PillowFight.cleanup(self) switch_to_default_rook_cluster_project()
class ElasticSearch(object): """ ElasticSearch Environment """ def __init__(self): """ Initializer function """ log.info("Initializing the Elastic-Search environment object") self.namespace = "elastic-system" self.eck_file = "ocs_ci/templates/app-pods/eck.1.3.1-all-in-one.yaml" self.pvc = "ocs_ci/templates/app-pods/es-pvc.yaml" self.crd = "ocs_ci/templates/app-pods/esq.yaml" self.lspid = None # Creating some different types of OCP objects self.ocp = OCP(kind="pod", resource_name="elastic-operator-0", namespace=self.namespace) self.ns_obj = OCP(kind="namespace", namespace=self.namespace) self.es = OCP(resource_name="quickstart-es-http", namespace=self.namespace) self.elasticsearch = OCP(namespace=self.namespace, kind="elasticsearch") self.password = OCP( kind="secret", resource_name="quickstart-es-elastic-user", namespace=self.namespace, ) # Deploy the ECK all-in-one.yaml file self._deploy_eck() # Deploy the Elastic-Search server self._deploy_es() # Verify that ES is Up & Running timeout = 600 while timeout > 0: if self.get_health(): log.info("The ElasticSearch server is ready !") break else: log.warning("The ElasticSearch server is not ready yet") log.info("going to sleep for 30 sec. before next check") time.sleep(30) timeout -= 30 # Starting LocalServer process - port forwarding self.local_server() # Connect to the server self.con = self._es_connect() def _deploy_eck(self): """ Deploying the ECK environment for the Elasticsearch, and make sure it is in Running mode """ log.info("Deploying the ECK environment for the ES cluster") self.ocp.apply(self.eck_file) for es_pod in TimeoutSampler(300, 10, get_pod_name_by_pattern, "elastic-operator", self.namespace): try: if es_pod[0] is not None: self.eckpod = es_pod[0] log.info(f"The ECK pod {self.eckpod} is ready !") break except IndexError: log.info("ECK operator pod not ready yet") def get_ip(self): """ This function return the IP address of the Elasticsearch cluster. this IP is to use inside the OCP cluster Return str : String that represent the Ip Address. """ return self.es.get()["spec"]["clusterIP"] def get_port(self): """ This function return the port of the Elasticsearch cluster. Return str : String that represent the port. """ return self.es.get()["spec"]["ports"][0]["port"] def _deploy_es(self): log.info("Deploy the PVC for the ElasticSearch cluster") self.ocp.apply(self.pvc) log.info("Deploy the ElasticSearch cluster") self.ocp.apply(self.crd) for es_pod in TimeoutSampler(300, 20, get_pod_name_by_pattern, "quickstart-es-default", self.namespace): try: if es_pod[0] is not None: self.espod = es_pod[0] log.info(f"The ElasticSearch pod {self.espod} Started") break except IndexError: log.info("elasticsearch pod not ready yet") es_pod = OCP(kind="pod", namespace=self.namespace) log.info("Waiting for ElasticSearch to Run") assert es_pod.wait_for_resource( condition=constants.STATUS_RUNNING, resource_name=self.espod, sleep=30, timeout=600, ) log.info("Elastic Search is ready !!!") def get_health(self): """ This method return the health status of the Elasticsearch. Returns: bool : True if the status is green (OK) otherwise - False """ return self.elasticsearch.get( )["items"][0]["status"]["health"] == "green" def get_password(self): """ This method return the password used to connect the Elasticsearch. Returns: str : The password as text """ return base64.b64decode( self.password.get()["data"]["elastic"]).decode("utf-8") def cleanup(self): """ Cleanup the environment from all Elasticsearch components, and from the port forwarding process. """ log.info("Teardown the Elasticsearch environment") log.info(f"Killing the local server process ({self.lspid})") os.kill(self.lspid, signal.SIGKILL) log.info("Deleting all resources") subprocess.run(f"oc delete -f {self.crd}", shell=True) self.ns_obj.delete_project(project_name=self.namespace) self.ns_obj.wait_for_delete(resource_name=self.namespace, timeout=180) def local_server(self): """ Starting sub-process that will do port-forwarding, to allow access from outside the open-shift cluster into the Elasticsearch server. """ cmd = f"oc -n {self.namespace } " cmd += f"port-forward service/quickstart-es-http {self.get_port()}" log.info(f"Going to run : {cmd}") proc = subprocess.Popen(cmd, shell=True) log.info(f"Starting LocalServer with PID of {proc.pid}") self.lspid = proc.pid def _es_connect(self): """ Create a connection to the ES via the localhost port-fwd Returns: Elasticsearch: elasticsearch connection object Raise: ConnectionError: if can not connect to the server """ try: es = Elasticsearch([{ "host": "localhost", "port": self.get_port() }]) except esexp.ConnectionError: log.error("Can not connect to ES server in the LocalServer") raise return es def get_indices(self): """ Getting list of all indices in the ES server - all created by the test, the installation of the ES was without any indexes pre-installed. Returns: list : list of all indices defined in the ES server """ results = [] log.info("Getting all indices") for ind in self.con.indices.get_alias("*"): results.append(ind) return results def _copy(self, es): """ Copy All data from the internal ES server to the main ES Args: es (obj): elasticsearch object which connected to the main ES """ query = {"size": 1000, "query": {"match_all": {}}} for ind in self.get_indices(): log.info(f"Reading {ind} from internal ES server") try: result = self.con.search(index=ind, body=query) except esexp.NotFoundError: log.warning(f"{ind} Not found in the Internal ES.") continue log.debug(f"The results from internal ES for {ind} are :{result}") log.info(f"Writing {ind} into main ES server") for doc in result["hits"]["hits"]: log.debug(f"Going to write : {doc}") es.index(index=ind, doc_type="_doc", body=doc["_source"])
class ElasticSearch(object): """ ElasticSearch Environment """ def __init__(self, **kwargs): """ Initializer function """ log.info("Initializing the Elastic-Search environment object") self.args = kwargs self.namespace = "elastic-system" self.repo = self.args.get("repo", constants.OCS_WORKLOADS) self.branch = self.args.get("branch", "master") self.dir = tempfile.mkdtemp(prefix="eck_") # Clone the ECK repo locally self._clone() self.eck_path = os.path.join(self.dir, "ocs-workloads/eck") self.eck_file = os.path.join(self.eck_path, "crds.yaml") self.dumper_file = os.path.join(constants.TEMPLATE_APP_POD_DIR, "esclient.yaml") self.crd = os.path.join(constants.TEMPLATE_APP_POD_DIR, "esq.yaml") # Creating some different types of OCP objects self.ocp = OCP(kind="pod", resource_name="elastic-operator-0", namespace=self.namespace) self.ns_obj = OCP(kind="namespace", namespace=self.namespace) # Verify that the namespace dose not exist, delete it if it is exists. if self.ns_obj.get(dont_raise=True, resource_name=self.namespace) is not None: log.warning( f"The {self.namespace} namespace is already exists!, try to delete it" ) self.ns_obj.delete_project(project_name=self.namespace) self.ns_obj.wait_for_delete(resource_name=self.namespace, timeout=180, sleep=5) self.es = OCP(resource_name="quickstart-es-http", namespace=self.namespace) self.elasticsearch = OCP(namespace=self.namespace, kind="elasticsearch") self.password = OCP( kind="secret", resource_name="quickstart-es-elastic-user", namespace=self.namespace, ) # Deploy the ECK all-in-one.yaml file self._deploy_eck() # Deploy the Elastic-Search server if not self._deploy_es(): self.cleanup() raise ElasticSearchNotDeployed("Elasticsearch deployment Failed") # Verify that ES is Up & Running sample = TimeoutSampler(timeout=180, sleep=10, func=self.get_health) if not sample.wait_for_func_status(True): self.cleanup() raise ElasticSearchNotDeployed("Elasticsearch deployment Failed") # Deploy the elasticsearch dumper pod self._deploy_data_dumper_client() # Connect to the server self.con = self._es_connect() def _clone(self): """ clone the ECK repo into temp directory """ try: log.info(f"Cloning ECK in {self.dir}") git_clone_cmd = f"git clone -b {self.branch} {self.repo} --depth 1" run(git_clone_cmd, shell=True, cwd=self.dir, check=True) except (CommandFailed, CalledProcessError) as cf: log.error("Error during cloning of ECK repository") raise cf def _pod_is_found(self, pattern): """ Boolean function which check if pod (by pattern) is exist. Args: pattern (str): the pattern of the pod to look for Returns: bool : True if pod found, otherwise False """ return len(get_pod_name_by_pattern(pattern, self.namespace)) > 0 def _deploy_eck(self): """ Deploying the ECK environment for the Elasticsearch, and make sure it is in Running mode """ log.info("Deploying the ECK environment for the ES cluster") log.info("Deploy the ECK CRD's") self.ocp.apply(self.eck_file) log.info("deploy the ECK operator") self.ocp.apply(f"{self.eck_path}/operator.yaml") sample = TimeoutSampler(timeout=300, sleep=10, func=self._pod_is_found, pattern="elastic-operator") if not sample.wait_for_func_status(True): err_msg = "ECK deployment Failed" log.error(err_msg) self.cleanup() raise Exception(err_msg) log.info("The ECK pod is ready !") def _deploy_data_dumper_client(self): """ Deploying elastic search client pod with utility which dump all the data from the server to .tgz file """ log.info("Deploying the es client for dumping all data") self.ocp.apply(self.dumper_file) sample = TimeoutSampler(timeout=300, sleep=10, func=self._pod_is_found, pattern="es-dumper") if not sample.wait_for_func_status(True): self.cleanup() raise Exception("Dumper pod deployment Failed") self.dump_pod = get_pod_name_by_pattern("es-dumper", self.namespace)[0] log.info(f"The dumper client pod {self.dump_pod} is ready !") def get_ip(self): """ This function return the IP address of the Elasticsearch cluster. this IP is to use inside the OCP cluster Return str : String that represent the Ip Address. """ return self.es.get()["spec"]["clusterIP"] def get_port(self): """ This function return the port of the Elasticsearch cluster. Return str : String that represent the port. """ return self.es.get()["spec"]["ports"][0]["port"] def _deploy_es(self): """ Deploying the Elasticsearch server """ # Creating PVC for the elasticsearch server and wait until it bound log.info("Creating 10 GiB PVC for the ElasticSearch cluster on") try: self.pvc_obj = create_pvc( sc_name=self.args.get("sc") or constants.CEPHBLOCKPOOL_SC, namespace=self.namespace, pvc_name="elasticsearch-data-quickstart-es-default-0", access_mode=constants.ACCESS_MODE_RWO, size="10Gi", ) # Make sure the PVC bound, or delete it and raise exception wait_for_resource_state(self.pvc_obj, constants.STATUS_BOUND) except ResourceWrongStatusException: log.error("The PVC couldn't created") return False self.pvc_obj.reload() log.info("Deploy the ElasticSearch cluster") self.ocp.apply(self.crd) sample = TimeoutSampler( timeout=300, sleep=10, func=self._pod_is_found, pattern="quickstart-es-default", ) if not sample.wait_for_func_status(True): log.error("The ElasticSearch pod deployment Failed") return False self.espod = get_pod_name_by_pattern("quickstart-es-default", self.namespace)[0] log.info(f"The ElasticSearch pod {self.espod} Started") es_pod = OCP(kind="pod", namespace=self.namespace) log.info("Waiting for ElasticSearch to Run") if not es_pod.wait_for_resource( condition=constants.STATUS_RUNNING, resource_name=self.espod, sleep=30, timeout=600, ): log.error("TThe ElasticSearch pod is not running !") return False else: log.info("Elastic Search is ready !!!") return True def get_health(self): """ This method return the health status of the Elasticsearch. Returns: bool : True if the status is green (OK) otherwise - False """ return self.elasticsearch.get( )["items"][0]["status"]["health"] == "green" def get_password(self): """ This method return the password used to connect the Elasticsearch. Returns: str : The password as text """ return base64.b64decode( self.password.get()["data"]["elastic"]).decode("utf-8") def cleanup(self): """ Cleanup the environment from all Elasticsearch components. """ log.info("Teardown the Elasticsearch environment") log.info("Deleting all resources") try: log.info("Deleting the dumper client pod") self.ocp.delete(yaml_file=self.dumper_file) except CommandFailed: # in case of the es-dumper did not deployed yet, trying to delete it # will failed. log.warning("es-dumper pod does not exist") pass try: log.info("Deleting the es resource") self.ocp.delete(yaml_file=self.crd) except CommandFailed: # in case of the elastic-search did not deployed yet, trying to # delete it will failed. log.warning("elastic-search pod does not exist") pass log.info("Deleting the es project") # self.ns_obj.delete_project(project_name=self.namespace) self.ocp.delete(f"{self.eck_path}/operator.yaml") self.ocp.delete(yaml_file=self.eck_file) self.ns_obj.wait_for_delete(resource_name=self.namespace, timeout=180) log.info("The ElasticSearch cleaned up from the cluster") def _es_connect(self): """ Create a connection to the local ES Returns: Elasticsearch: elasticsearch connection object, None if Cannot connect to ES """ try: es = Elasticsearch([{ "host": self.get_ip(), "port": self.get_port() }]) except esexp.ConnectionError: log.warning("Cannot connect to ES server in the LocalServer") es = None return es def get_indices(self): """ Getting list of all indices in the ES server - all created by the test, the installation of the ES was without any indexes pre-installed. Returns: list : list of all indices defined in the ES server """ results = [] log.info("Getting all indices") for ind in self.con.indices.get_alias("*"): results.append(ind) return results def dumping_all_data(self, target_path): """ Dump All data from the internal ES server to .tgz file. Args: target_path (str): the path where the results file will be copy into Return: bool: True if the dump operation succeed and return the results data to the host otherwise False """ log.info("dumping data from ES server to .tgz file") rsh_cmd = f"rsh {self.dump_pod} /elasticsearch-dump/esdumper.py --ip {self.get_ip()} --port {self.get_port()}" result = self.ocp.exec_oc_cmd(rsh_cmd, out_yaml_format=False, timeout=1200) if "ES dump is done." not in result: log.error("There is no data in the Elasticsearch server") return False else: src_file = result.split()[-1] log.info(f"Copy {src_file} from the client pod") cp_command = f"cp {self.dump_pod}:{src_file} {target_path}/FullResults.tgz" result = self.ocp.exec_oc_cmd(cp_command, timeout=120) log.info(f"The output from the POD is {result}") log.info("Extracting the FullResults.tgz file") kwargs = {"cwd": target_path} results = run_command(f"tar zxvf {target_path}/FullResults.tgz", **kwargs) log.debug(f"The untar results is {results}") if "Error in command" in results: log.warning("Cannot untar the dumped file") return False return True