def delete(self, yaml_file=None, resource_name='', wait=True, force=False): """ Deletes a resource Args: yaml_file (str): Path to a yaml file to use in 'oc delete -f file.yaml resource_name (str): Name of the resource you want to delete wait (bool): Determines if the delete command should wait to completion force (bool): True for force deletion with --grace-period=0, False otherwise Returns: dict: Dictionary represents a returned yaml file Raises: CommandFailed: In case yaml_file and resource_name wasn't provided """ if not (yaml_file or resource_name): raise CommandFailed( "At least one of resource_name or yaml_file have to " "be provided") command = f"delete " if resource_name: command += f"{self.kind} {resource_name}" else: command += f"-f {yaml_file}" if force: command += " --grace-period=0 --force" # oc default for wait is True if not wait: command += " --wait=false" return self.exec_oc_cmd(command)
def run_cmd(cmd, **kwargs): """ Run an arbitrary command locally Args: cmd (str): command to run Raises: CommandFailed: In case the command execution fails Returns: (str) Decoded stdout of command """ log.info(f"Executing command: {cmd}") if isinstance(cmd, str): cmd = shlex.split(cmd) r = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, **kwargs) log.debug(f"Command output: {r.stdout.decode()}") if r.stderr and not r.returncode: log.warning(f"Command warning:: {r.stderr.decode()}") if r.returncode: raise CommandFailed(f"Error during execution of command: {cmd}." f"\nError is {r.stderr.decode()}") return r.stdout.decode()
def create(self, yaml_file=None, resource_name='', out_yaml_format=True): """ Creates a new resource Args: yaml_file (str): Path to a yaml file to use in 'oc create -f file.yaml resource_name (str): Name of the resource you want to create out_yaml_format (bool): Determines if the output should be formatted to a yaml like string Returns: dict: Dictionary represents a returned yaml file """ if not (yaml_file or resource_name): raise CommandFailed( "At least one of resource_name or yaml_file have to " "be provided") command = "create " if yaml_file: command += f"-f {yaml_file}" elif resource_name: # e.g "oc namespace my-project" command += f"{self.kind} {resource_name}" if out_yaml_format: command += " -o yaml" output = self.exec_oc_cmd(command) log.debug(f"{yaml.dump(output)}") return output
def exec_ceph_cmd(self, ceph_cmd, format='json-pretty'): """ Execute a Ceph command on the Ceph tools pod Args: ceph_cmd (str): The Ceph command to execute on the Ceph tools pod format (str): The returning output format of the Ceph command Returns: dict: Ceph command output Raises: CommandFailed: In case the pod is not a toolbox pod """ if 'rook-ceph-tools' not in self.labels.values(): raise CommandFailed( "Ceph commands can be executed only on toolbox pod") ceph_cmd = ceph_cmd if format: ceph_cmd += f" --format {format}" out = self.exec_cmd_on_pod(ceph_cmd) # For some commands, like "ceph fs ls", the returned output is a list if isinstance(out, list): return [item for item in out if item] return out
def setup(self): """ Setting up the test environment : Calculating the amount of storage which available for the test Creating namespace (project) for the test """ log.info("Setting up the test environment") super(TestPvcMultiSnapshotPerformance, self).setup() # Getting the total Storage capacity try: self.ceph_capacity = int(self.ceph_cluster.get_ceph_capacity()) except Exception as err: err_msg = f"Failed to get Storage capacity : {err}" log.error(err_msg) raise Exception(err_msg) # Use 70% of the storage capacity in the test self.capacity_to_use = int(self.ceph_capacity * 0.7) # Creating new namespace for the test self.nss_name = "pas-test-namespace" log.info(f"Creating new namespace ({self.nss_name}) for the test") try: self.proj = helpers.create_project(project_name=self.nss_name) except CommandFailed as ex: if str(ex).find("(AlreadyExists)"): log.warning("The Namespace is Already Exists !") log.error("Can not create new project") raise CommandFailed(f"{self.nss_name} was not created") # Initialize a general Snapshot object to use in the test self.snapshot = OCP(kind="volumesnapshot", namespace=self.nss_name)
def generate_onboarding_token(): """ Generate Onboarding token for consumer cluster via following steps: 1. Download ticketgen.sh script from: https://raw.githubusercontent.com/jarrpa/ocs-operator/ticketgen/hack/ticketgen/ticketgen.sh 2. Save private key from AUTH["managed_service"]["private_key"] to temporary file. 3. Run ticketgen.sh script to generate Onboarding token. Raises: CommandFailed: In case the script ticketgen.sh fails. ConfigurationError: when AUTH["managed_service"]["private_key"] not is not defined Returns: string: Onboarding token """ logger.debug("Generate onboarding token for ODF to ODF deployment") ticketgen_script_path = os.path.join(constants.DATA_DIR, "ticketgen.sh") # download ticketgen.sh script logger.debug("Download and prepare ticketgen.sh script") download_file( "https://raw.githubusercontent.com/jarrpa/ocs-operator/ticketgen/hack/ticketgen/ticketgen.sh", ticketgen_script_path, ) # add execute permission to the ticketgen.sh script current_file_permissions = os.stat(ticketgen_script_path) os.chmod( ticketgen_script_path, current_file_permissions.st_mode | stat.S_IEXEC, ) # save private key to temp file logger.debug("Prepare temporary file with private key") private_key = config.AUTH.get("managed_service", {}).get("private_key", "") if not private_key: raise ConfigurationError( "Private key for Managed Service not defined.\n" "Expected following configuration in auth.yaml file:\n" "managed_service:\n" ' private_key: "..."\n' ' public_key: "..."' ) with NamedTemporaryFile( mode="w", prefix="private", suffix=".pem", delete=True ) as key_file: key_file.write(private_key) key_file.flush() logger.debug("Generate Onboarding token") ticketgen_result = exec_cmd(f"{ticketgen_script_path} {key_file.name}") ticketgen_output = ticketgen_result.stdout.decode() if ticketgen_result.stderr: raise CommandFailed( f"Script ticketgen.sh failed to generate Onboarding token:\n" f"command: '{' '.join(ticketgen_result.args)}'\n" f"stderr: {ticketgen_result.stderr.decode()}\n" f"stdout: {ticketgen_output}" ) return ticketgen_output
def create_test_project(self): """ Creating new project (namespace) for performance test """ self.namespace = helpers.create_unique_resource_name("pas-test", "namespace") log.info(f"Creating new namespace ({self.namespace}) for the test") try: self.proj = helpers.create_project(project_name=self.namespace) except CommandFailed as ex: if str(ex).find("(AlreadyExists)"): log.warning("The namespace already exists !") log.error("Cannot create new project") raise CommandFailed(f"{self.namespace} was not created")
def delete_test_project(self): """ Deleting the performance test project (namespace) """ log.info(f"Deleting the test namespace : {self.namespace}") switch_to_default_rook_cluster_project() try: self.proj.delete(resource_name=self.namespace) self.proj.wait_for_delete( resource_name=self.namespace, timeout=60, sleep=10 ) except CommandFailed: log.error(f"Cannot delete project {self.namespace}") raise CommandFailed(f"{self.namespace} was not created")
def delete_project(self, project_name): """ Delete a project. A project created by the new_project function does not have a corresponding yaml file so normal resource deletion calls do not work Args: project_name (str): Name of the project to be deleted Returns: bool: True in case project deletion succeeded. Raises: CommandFailed: When the project deletion does not succeed. """ command = f"oc delete project {project_name}" if f' "{project_name}" deleted' in run_cmd(f"{command}"): return True raise CommandFailed(f"{project_name} was not deleted")
def run_cmd(cmd, secrets=None, timeout=600, ignore_error=False, **kwargs): """ Run an arbitrary command locally Args: cmd (str): command to run secrets (list): A list of secrets to be masked with asterisks This kwarg is popped in order to not interfere with subprocess.run(``**kwargs``) timeout (int): Timeout for the command, defaults to 600 seconds. ignore_error (bool): True if ignore non zero return code and do not raise the exception. Raises: CommandFailed: In case the command execution fails Returns: (str) Decoded stdout of command """ masked_cmd = mask_secrets(cmd, secrets) log.info(f"Executing command: {masked_cmd}") if isinstance(cmd, str): cmd = shlex.split(cmd) r = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, timeout=timeout, **kwargs ) log.debug(f"Command output: {r.stdout.decode()}") if r.stderr and not r.returncode: log.warning(f"Command warning: {mask_secrets(r.stderr.decode(), secrets)}") if r.returncode and not ignore_error: raise CommandFailed( f"Error during execution of command: {masked_cmd}." f"\nError is {mask_secrets(r.stderr.decode(), secrets)}" ) return mask_secrets(r.stdout.decode(), secrets)
def get_fio_results(self): """ Get FIO execution results Returns: dict: Dictionary represents the FIO execution results Raises: Exception: In case of exception from FIO """ try: result = self.fio_thread.result(FIO_TIMEOUT) if result: return yaml.safe_load(result) raise CommandFailed(f"FIO execution results: {result}.") except CommandFailed as ex: logger.exception(f"FIO failed: {ex}") raise except Exception as ex: logger.exception(f"Found Exception: {ex}") raise
def run_cmd(cmd, secrets=None, **kwargs): """ Run an arbitrary command locally Args: cmd (str): command to run secrets (list): A list of secrets to be masked with asterisks This kwarg is popped in order to not interfere with subprocess.run(**kwargs) Raises: CommandFailed: In case the command execution fails Returns: (str) Decoded stdout of command """ masked_cmd = mask_secrets(cmd, secrets) log.info(f"Executing command: {masked_cmd}") if isinstance(cmd, str): cmd = shlex.split(cmd) r = subprocess.run( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE, **kwargs ) log.debug(f"Command output: {r.stdout.decode()}") if r.stderr and not r.returncode: log.warning(f"Command warning: {mask_secrets(r.stderr.decode(), secrets)}") if r.returncode: raise CommandFailed( f"Error during execution of command: {masked_cmd}." f"\nError is {mask_secrets(r.stderr.decode(), secrets)}" ) return mask_secrets(r.stdout.decode(), secrets)
def win_exec(self, ps_command, timeout=180): log.info("Running powershell`s command `{}`".format(ps_command)) client = paramiko.SSHClient() client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) client.connect(self.ip_address, username=self.login, password=self.password) command = 'powershell -Command "& {{{}}}"'.format(ps_command) chan_ssh = client.get_transport().open_session() chan_ssh.exec_command(command) for i in range(0, timeout): sleep(1) if chan_ssh.exit_status_ready(): break else: raise TimeoutException("Timeout") output = dict() output["exit_code"] = chan_ssh.recv_exit_status() output["stdout"] = chan_ssh.recv(-1) output["stderr"] = chan_ssh.recv_stderr(-1) if not bool(output["stderr"]) and output["exit_code"] == 0: return output else: raise CommandFailed(output["stderr"])
def run(self, podname, namespace, cmd_obj): resp = None stdout = None stderr = None ret = None try: resp = self.api.read_namespaced_pod(name=podname, namespace=namespace) logger.info(resp) except ApiException as ex: if ex.status != 404: logger.error("Unknown error: %s" % ex) # run command in bash bash = ["/bin/bash"] resp = stream( self.api.connect_get_namespaced_pod_exec, podname, namespace, command=bash, stderr=True, stdin=True, stdout=True, tty=False, _preload_content=False, ) done = False outbuf = "" while resp.is_open(): resp.update(timeout=1) if resp.peek_stdout(): stdout = resp.read_stdout(timeout=cmd_obj.timeout) outbuf = outbuf + stdout if cmd_obj.long_running: while resp.peek_stdout(timeout=cmd_obj.timeout): stdout = resp.read_stdout(timeout=cmd_obj.timeout) outbuf = outbuf + stdout if resp.peek_stderr(): stderr = resp.read_stderr(timeout=60) if not done: resp.write_stdin(cmd_obj.cmd) resp.write_stdin("\n") done = True else: break """ Couple of glitches in capturing return value. Rest api doesn't return ret value of the command hence this workaround. we can fix this once we have facility to capture err code """ if cmd_obj.check_ec: resp.write_stdin("echo $?\n") try: ret = int(resp.readline_stdout(timeout=5)) except (ValueError, TypeError): logger.error(f"TimeOut: Command timedout after {cmd_obj.timeout}") raise CommandFailed(f'Failed to run "{cmd_obj.cmd}"') finally: resp.close() if outbuf: stdout = outbuf return stdout, stderr, ret
def teardown(self): """ Cleaning up the environment : Delete all snapshot Delete the POD Delete the PVC and the PV Delete the StorageClass Delete the VolumeSnapshotClass Delete the data pool Switch to the default namespace Delete the tested namespace """ log.info("Cleanup the test environment") if self.full_teardown: # Getting the name of the PCV's backed PV try: pv = self.pvc_obj.get("spec")["spec"]["volumeName"] except KeyError: log.error( f"Cannot found key in the PVC object {json.dumps(self.pvc_obj.get('spec').get('spec'), indent=3)}" ) # Getting the list of all snapshots try: snapshot_list = self.snapshot.get(all_namespaces=True)["items"] except Exception as err: log.error(f"Cannot get the list of snapshots : {err}") snapshot_list = [] # Deleting al snapshots from the cluster log.info(f"Trying to delete all ({len(snapshot_list)}) Snapshots") log.debug( f"The list of all snapshots is : {json.dumps(snapshot_list, indent=3)}" ) for vs in snapshot_list: snap_name = vs["metadata"]["name"] log.info(f"Try to delete {snap_name}") try: self.snapshot.delete(resource_name=snap_name) except Exception as err: log.error(f"Cannot delete {snap_name} : {err}") # Deleting the pod which wrote data to the pvc log.info(f"Deleting the test POD : {self.pod_obj.name}") try: self.pod_obj.delete() log.info("Wait until the pod is deleted.") self.pod_obj.ocp.wait_for_delete( resource_name=self.pod_obj.name) except Exception as ex: log.error(f"Cannot delete the test pod : {ex}") # Deleting the PVC which used in the test. try: log.info(f"Delete the PVC : {self.pvc_obj.name}") self.pvc_obj.delete() log.info("Wait until the pvc is deleted.") self.pvc_obj.ocp.wait_for_delete( resource_name=self.pvc_obj.name) except Exception as ex: log.error(f"Cannot delete the test pvc : {ex}") # Delete the backend PV of the PVC log.info(f"Try to delete the backend PV : {pv}") try: run_oc_command(f"delete pv {pv}") except Exception as ex: err_msg = f"cannot delete PV {pv} - [{ex}]" log.error(err_msg) # Deleting the StorageClass used in the test log.info(f"Deleting the test StorageClass : {self.sc_obj.name}") try: self.sc_obj.delete() log.info("Wait until the SC is deleted.") self.sc_obj.ocp.wait_for_delete(resource_name=self.sc_obj.name) except Exception as ex: log.error(f"Can not delete the test sc : {ex}") # Deleting the VolumeSnapshotClass used in the test log.info( f"Deleting the test Snapshot Class : {self.snap_class.name}") try: self.snap_class.delete() log.info("Wait until the VSC is deleted.") self.snap_class.ocp.wait_for_delete( resource_name=self.snap_class.name) except Exception as ex: log.error(f"Can not delete the test vsc : {ex}") # Deleting the Data pool log.info(f"Deleting the test storage pool : {self.sc_name}") self.delete_ceph_pool(self.sc_name) # Verify deletion by checking the backend CEPH pools using the toolbox results = self.ceph_cluster.toolbox.exec_cmd_on_pod( "ceph osd pool ls") log.debug(f"Existing pools are : {results}") if self.sc_name in results.split(): log.warning( "The pool did not deleted by CSI, forcing delete it manually" ) self.ceph_cluster.toolbox.exec_cmd_on_pod( f"ceph osd pool delete {self.sc_name} {self.sc_name} " "--yes-i-really-really-mean-it") else: log.info(f"The pool {self.sc_name} was deleted successfully") # Deleting the namespace used by the test log.info(f"Deleting the test namespace : {self.nss_name}") switch_to_default_rook_cluster_project() try: self.proj.delete(resource_name=self.nss_name) self.proj.wait_for_delete(resource_name=self.nss_name, timeout=60, sleep=10) except CommandFailed: log.error(f"Can not delete project {self.nss_name}") raise CommandFailed(f"{self.nss_name} was not created") # After deleting all data from the cluster, we need to wait until it will re-balance ceph_health_check(namespace=constants.OPENSHIFT_STORAGE_NAMESPACE, tries=30, delay=60) super(TestPvcMultiSnapshotPerformance, self).teardown()
def exec_command(self, **kw): """ execute a command on the vm eg: self.exec_cmd(cmd='uptime') or self.exec_cmd(cmd='background_cmd', check_ec=False) Attributes: check_ec (bool): False will run the command and not wait for exit code """ if kw.get("sudo"): ssh = self.rssh else: ssh = self.ssh if kw.get("timeout"): timeout = kw["timeout"] else: timeout = 120 logger.info("Running command %s on %s", kw["cmd"], self.ip_address) stdin = None stdout = None stderr = None if self.run_once: self.ssh_transport().set_keepalive(15) self.rssh_transport().set_keepalive(15) if kw.get("long_running"): logger.info("long running command --") channel = ssh().get_transport().open_session() channel.exec_command(kw["cmd"]) read = "" while True: if channel.exit_status_ready(): ec = channel.recv_exit_status() break rl, wl, xl = select([channel], [], [channel], 4200) if len(rl) > 0: data = channel.recv(1024) read += data.decode() logger.info(data.decode()) if len(xl) > 0: data = channel.recv(1024) read += data.decode() logger.info(data.decode()) return read, ec try: stdin, stdout, stderr = ssh().exec_command(kw["cmd"], timeout=timeout) except SSHException as e: logger.error("Exception during cmd %s", str(e)) if "Timeout openning channel" in str(e): logger.error("channel reset error") exit_status = stdout.channel.recv_exit_status() self.exit_status = exit_status if kw.get("check_ec", True): if exit_status == 0: logger.info("Command completed successfully") else: logger.error("Error during cmd %s, timeout %d", exit_status, timeout) raise CommandFailed(kw["cmd"] + " Error: " + str(stderr.read().decode()) + " " + str(self.ip_address)) return stdout, stderr else: return stdout, stderr