def finalizer(): """ Delete bdi temporary directory """ log.info(f"Deleting directory {bdi_dir[0]}") # if os.path.isdir(class_instance.bdi_dir[0]): exec_cmd(cmd=f"rm -rf {bdi_dir[0]}")
def delete_htpasswd_secret(): """ Delete HTPasswd secret. """ cmd = "oc delete secret htpass-secret -n openshift-config" exec_cmd(cmd)
def prepare_monstore_script(self): """ Prepares the script to retrieve the `monstore` cluster map from OSDs """ recover_mon = """ #!/bin/bash ms=/tmp/monstore rm -rf $ms mkdir $ms for osd_pod in $(oc get po -l app=rook-ceph-osd -oname -n openshift-storage); do echo "Starting with pod: $osd_pod" podname=$(echo $osd_pod| cut -c5-) oc exec $osd_pod -- rm -rf $ms oc cp $ms $podname:$ms rm -rf $ms mkdir $ms dp=/var/lib/ceph/osd/ceph-$(oc get $osd_pod -ojsonpath='{ .metadata.labels.ceph_daemon_id }') op=update-mon-db ot=ceph-objectstore-tool echo "pod in loop: $osd_pod ; done deleting local dirs" oc exec $osd_pod -- $ot --type bluestore --data-path $dp --op $op --no-mon-config --mon-store-path $ms echo "Done with COT on pod: $osd_pod" oc cp $podname:$ms $ms echo "Finished pulling COT data from pod: $osd_pod" done """ with open(f"{self.backup_dir}/recover_mon.sh", "w") as file: file.write(recover_mon) exec_cmd(cmd=f"chmod +x {self.backup_dir}/recover_mon.sh")
def exec_mcg_cmd(self, cmd, namespace=None, use_yes=False, **kwargs): """ Executes an MCG CLI command through the noobaa-operator pod's CLI binary Args: cmd (str): The command to run namespace (str): The namespace to run the command in Returns: str: stdout of the command """ kubeconfig = os.getenv("KUBECONFIG") if kubeconfig: kubeconfig = f"--kubeconfig {kubeconfig} " namespace = f"-n {namespace}" if namespace else f"-n {self.namespace}" if use_yes: result = exec_cmd( [ f"yes | {constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH} {cmd} {namespace}" ], shell=True, **kwargs, ) else: result = exec_cmd( f"{constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH} {cmd} {namespace}", **kwargs, ) result.stdout = result.stdout.decode() result.stderr = result.stderr.decode() return result
def set_pagerduty_integration_secret(integration_key): """ Update ocs-converged-pagerduty secret. This is valid only on ODF Managed Service. ocs-converged-pagerduty secret is expected to be present prior to the update. Args: integration_key (str): Integration key taken from PagerDuty Prometheus integration """ logger.info("Setting up PagerDuty integration") kubeconfig = os.getenv("KUBECONFIG") cmd = ( f"oc create secret generic {managedservice.get_pagerduty_secret_name()} " f"--from-literal=PAGERDUTY_KEY={integration_key} -n openshift-storage " f"--kubeconfig {kubeconfig} --dry-run -o yaml") secret_data = exec_cmd( cmd, secrets=[ integration_key, managedservice.get_pagerduty_secret_name(), ], ).stdout with tempfile.NamedTemporaryFile( prefix=f"{managedservice.get_pagerduty_secret_name()}_" ) as secret_file: secret_file.write(secret_data) secret_file.flush() exec_cmd(f"oc apply --kubeconfig {kubeconfig} -f {secret_file.name}") logger.info("New integration key was set.")
def clone_and_unlock_ocs_private_conf(self): """ Clone ocs_private_conf (flexy env and config) repo into flexy_host_dir """ clone_repo( self.flexy_private_conf_url, self.flexy_host_private_conf_dir_path, self.flexy_private_conf_branch, ) # prepare flexy private repo keyfile (if it is base64 encoded) keyfile = os.path.expanduser(constants.FLEXY_GIT_CRYPT_KEYFILE) try: with open(keyfile, "rb") as fd: keyfile_content = base64.b64decode(fd.read()) logger.info( "Private flexy repository git crypt keyfile is base64 encoded. " f"Decoding it and saving to the same place ({keyfile})") with open(keyfile, "wb") as fd: fd.write(keyfile_content) except binascii.Error: logger.info( f"Private flexy repository git crypt keyfile is already prepared ({keyfile})." ) # git-crypt unlock /path/to/keyfile cmd = f"git-crypt unlock {keyfile}" exec_cmd(cmd, cwd=self.flexy_host_private_conf_dir_path) logger.info("Unlocked the git repo")
def flexy_prepare_work_dir(self): """ Prepare Flexy working directory (flexy-dir): - copy flexy-dir from cluster_path to data dir (if available) - set proper ownership """ logger.info(f"Prepare flexy working directory {self.flexy_host_dir}.") if not os.path.exists(self.flexy_host_dir): # if ocs-ci/data were cleaned up (e.g. on Jenkins) and flexy-dir # exists in cluster dir, copy it to the data directory, othervise # just create empty flexy-dir cluster_path_flexy_dir = os.path.join(self.cluster_path, constants.FLEXY_HOST_DIR) if os.path.exists(cluster_path_flexy_dir): shutil.copytree( cluster_path_flexy_dir, self.flexy_host_dir, symlinks=True, ignore_dangling_symlinks=True, ) else: os.mkdir(self.flexy_host_dir) # change the ownership to the uid of user in flexy container chown_cmd = ( f"sudo chown -R {constants.FLEXY_USER_LOCAL_UID} {self.flexy_host_dir}" ) exec_cmd(chown_cmd)
def get_csv_from_image(bundle_image): """ Extract clusterserviceversion.yaml file from operator bundle image. Args: bundle_image (str): OCS operator bundle image Returns: dict: loaded yaml from CSV file """ manifests_dir = os.path.join(config.ENV_DATA["cluster_path"], constants.MANIFESTS_DIR) ocs_operator_csv_yaml = os.path.join(manifests_dir, constants.OCS_OPERATOR_CSV_YAML) create_directory_path(manifests_dir) with prepare_customized_pull_secret(bundle_image) as authfile_fo: exec_cmd( f"oc image extract --registry-config {authfile_fo.name} " f"{bundle_image} --confirm " f"--path /manifests/ocs-operator.clusterserviceversion.yaml:{manifests_dir}" ) try: with open(ocs_operator_csv_yaml) as f: return yaml.safe_load(f) except FileNotFoundError as err: logger.error(f"File {ocs_operator_csv_yaml} does not exists ({err})") raise
def create_htpasswd_idp(): """ Create OAuth identity provider of HTPasswd type. It uses htpass-secret secret as a source for list of users. """ cmd = f"oc apply -f {constants.HTPASSWD_IDP_YAML}" exec_cmd(cmd)
def finalizer(): """ Delete Security Context Constraints """ ocp_project = ocp.OCP() ocp_project.exec_oc_cmd(command="delete scc db2wh-scc") if os.path.isfile(temp_scc_yaml[0].name): exec_cmd(cmd="rm -f " + temp_scc_yaml[0].name)
def get_opm_tool(): """ Download and install opm tool. """ try: opm_version = exec_cmd("opm version") except (CommandFailed, FileNotFoundError): logger.info("opm tool is not available, installing it") opm_release_tag = config.ENV_DATA.get("opm_release_tag", "latest") if opm_release_tag != "latest": opm_release_tag = f"tags/{opm_release_tag}" opm_releases_api_url = ( f"https://api.github.com/repos/{config.ENV_DATA.get('opm_owner_repo')}/" f"releases/{opm_release_tag}") if config.AUTH.get("github"): github_auth = ( config.AUTH["github"].get("username"), config.AUTH["github"].get("token"), ) logger.debug( f"Using github authentication (user: {github_auth[0]})") else: github_auth = None logger.warning( "Github credentials are not provided in data/auth.yaml file. " "You might encounter issues with accessing github api as it " "have very strict rate limit for unauthenticated requests " "(60 requests per hour). Please check docs/getting_started.md " "file to find how to configure github authentication.") release_data = json.loads( get_url_content(opm_releases_api_url, auth=github_auth)) if platform.system() == "Darwin": opm_asset_name = "darwin-amd64-opm" elif platform.system() == "Linux": opm_asset_name = "linux-amd64-opm" else: raise UnsupportedOSType for asset in release_data["assets"]: if asset["name"] == opm_asset_name: opm_download_url = asset["browser_download_url"] break else: raise NotFoundError( f"opm binary for selected type {opm_asset_name} was not found") prepare_bin_dir() bin_dir = os.path.expanduser(config.RUN["bin_dir"]) logger.info( f"Downloading opm tool from '{opm_download_url}' to '{bin_dir}'") download_file(opm_download_url, os.path.join(bin_dir, "opm")) cmd = f"chmod +x {os.path.join(bin_dir, 'opm')}" exec_cmd(cmd) opm_version = exec_cmd("opm version") logger.info(f"opm tool is available: {opm_version.stdout.decode('utf-8')}")
def quay_super_user_login(endpoint_url): """ Logins in to quay endpoint Args: endpoint_url (str): External endpoint of quay """ exec_cmd( f"podman login {endpoint_url} -u {constants.QUAY_SUPERUSER} -p {constants.QUAY_PW} --tls-verify=false" )
def retrieve_noobaa_cli_binary(self): """ Copy the NooBaa CLI binary from the operator pod if it wasn't found locally, or if the hashes between the two don't match. Raises: NoobaaCliChecksumFailedException: If checksum doesn't match. AssertionError: In the case CLI binary doesn't exist. """ def _compare_cli_hashes(): """ Verify that the remote and local CLI binaries are the same in order to make sure the local bin is up to date Returns: bool: Whether the local and remote hashes are identical """ remote_cli_bin_md5 = cal_md5sum( self.operator_pod, constants.NOOBAA_OPERATOR_POD_CLI_PATH) logger.info(f"Remote noobaa cli md5 hash: {remote_cli_bin_md5}") local_cli_bin_md5 = calc_local_file_md5_sum( constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH) logger.info(f"Local noobaa cli md5 hash: {local_cli_bin_md5}") return remote_cli_bin_md5 == local_cli_bin_md5 if (not os.path.isfile(constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH) or not _compare_cli_hashes()): local_mcg_cli_dir = os.path.dirname( constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH) remote_mcg_cli_basename = os.path.basename( constants.NOOBAA_OPERATOR_POD_CLI_PATH) # BZ: https://bugzilla.redhat.com/show_bug.cgi?id=2011845 # using rsync instead of cp is more reliable cmd = (f"oc rsync -n {self.namespace} {self.operator_pod.name}:" f"{constants.NOOBAA_OPERATOR_POD_CLI_PATH}" f" {local_mcg_cli_dir}") subprocess.run(cmd, shell=True) exec_cmd(cmd) os.rename( os.path.join(local_mcg_cli_dir, remote_mcg_cli_basename), constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH, ) # Make sure the binary was copied properly and has the correct permissions assert os.path.isfile( constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH ), f"MCG CLI file not found at {constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH}" assert os.access( constants.NOOBAA_OPERATOR_LOCAL_CLI_PATH, os.X_OK ), "The MCG CLI binary does not have execution permissions" if not _compare_cli_hashes(): raise NoobaaCliChecksumFailedException( "Binary hash doesn't match the one on the operator pod")
def link_sa_and_secret(sa_name, secret_name, namespace): """ Link service account and secret for pulling of images. Args: sa_name (str): service account name secret_name (str): secret name namespace (str): namespace name """ exec_cmd(f"oc secrets link {sa_name} {secret_name} --for=pull -n {namespace}")
def test_user_creation(user_factory): user = user_factory() kubeconfig = os.getenv('KUBECONFIG') kube_data = "" with open(kubeconfig, 'r') as kube_file: kube_data = kube_file.readlines() sleep(30) exec_cmd(['oc', 'login', '-u', user[0], '-p', user[1]], secrets=[user[1]]) exec_cmd(['oc', 'logout']) with open(kubeconfig, 'w') as kube_file: kube_file.writelines(kube_data)
def test_user_creation(user_factory): user = user_factory() kubeconfig = os.getenv("KUBECONFIG") kube_data = "" with open(kubeconfig, "r") as kube_file: kube_data = kube_file.readlines() sleep(30) exec_cmd(["oc", "login", "-u", user[0], "-p", user[1]], secrets=[user[1]]) exec_cmd(["oc", "logout"]) with open(kubeconfig, "w") as kube_file: kube_file.writelines(kube_data)
def finalizer(): """ Delete 'tiller' project and temporary files """ ocp_project = ocp.OCP(kind="Project", namespace=tiller_namespace) ocp.switch_to_project("openshift-storage") log.info(f"Deleting project {tiller_namespace}") ocp_project.delete_project(project_name=tiller_namespace) ocp_project.wait_for_delete(resource_name=tiller_namespace) if os.path.isdir(helm_dir): exec_cmd(cmd="rm -rf " + helm_dir)
def flexy_backup_work_dir(self): """ Perform copying of flexy-dir to cluster_path. """ # change ownership of flexy-dir back to current user chown_cmd = f"sudo chown -R {os.getuid()}:{os.getgid()} {self.flexy_host_dir}" exec_cmd(chown_cmd) chmod_cmd = f"sudo chmod -R a+rX {self.flexy_host_dir}" exec_cmd(chmod_cmd) # mirror flexy work dir to cluster path rsync_cmd = f"rsync -av {self.flexy_host_dir} {self.cluster_path}/" exec_cmd(rsync_cmd, timeout=1200) # mirror install-dir to cluster path (auth directory, metadata.json # file and other files) install_dir = os.path.join(self.flexy_host_dir, constants.FLEXY_RELATIVE_CLUSTER_DIR) rsync_cmd = f"rsync -av {install_dir}/ {self.cluster_path}/" exec_cmd(rsync_cmd) if config.ENV_DATA["platform"].lower() == constants.VSPHERE_PLATFORM: # copy terraform.tfvars and terraform.tfstate files to # terraform_data directory in cluster path flexy_terraform_dir = os.path.join( self.flexy_host_dir, constants.FLEXY_RELATIVE_CLUSTER_DIR, ) terraform_data_dir = os.path.join(self.cluster_path, constants.TERRAFORM_DATA_DIR) for _file in ("terraform.tfstate", "terraform.tfvars"): shutil.copy2(os.path.join(flexy_terraform_dir, _file), terraform_data_dir)
def generate_kubeconfig_file(self, path=None, skip_tls_verify=False): """ creates kubeconfig file for the cluster Args: path (str): Path to create kubeconfig file skip_tls_verify (bool): True to bypass the certificate check and use insecure connections """ path = path or self.kubeconfig_path cmd = f"{config.ENV_DATA['ms_prod_oc_login']} --kubeconfig {path}" if skip_tls_verify: cmd = f"{cmd} --insecure-skip-tls-verify" exec_cmd(cmd)
def start(self, node, timeout): """ Start the given service using systemctl. Args: node (object): Node objects timeout (int): time in seconds to wait for service to start. Raises: UnexpectedBehaviour: If service on powerNode machine is still not up """ nodeip = self.nodes[node.name] cmd = f"ssh core@{nodeip} sudo systemctl start {self.service_name}.service" result = exec_cmd(cmd) logger.info( f"Result of start of service {self.service_name} is {result}") ret = TimeoutSampler( timeout=timeout, sleep=3, func=self.verify_service, node=node, action=ACTIVE, ) if not ret.wait_for_func_status(result=True): raise UnexpectedBehaviour( "Service {self.service_name} on Node {node.name} is still not Running" )
def start_powernodes_machines(self, powernode_machines, timeout=900, wait=True, force=True): """ Start PowerNode Machines Args: powernode_machines (list): List of PowerNode machines timeout (int): time in seconds to wait for node to reach 'not ready' state, and 'ready' state. wait (bool): Wait for PowerNodes to start - for future use force (bool): True for PowerNode ungraceful power off, False for graceful PowerNode shutdown - for future use """ ocpversion = get_ocp_version("-") for node in powernode_machines: result = exec_cmd( f"sudo virsh start test-ocp{ocpversion}-{node.name}") logger.info(f"Result of shutdown {result}") wait_for_cluster_connectivity(tries=900) wait_for_nodes_status(node_names=get_master_nodes(), status=constants.NODE_READY, timeout=timeout) wait_for_nodes_status(node_names=get_worker_nodes(), status=constants.NODE_READY, timeout=timeout)
def stop_powernodes_machines(self, powernode_machines, timeout=900, wait=True, force=True): """ Stop PowerNode Machines Args: powernode_machines (list): PowerNode objects timeout (int): time in seconds to wait for node to reach 'not ready' state wait (bool): True if need to wait till the restarted node reaches timeout - for future use force (bool): True for PowerNode ungraceful power off, False for graceful PowerNode shutdown - for future use Raises: UnexpectedBehaviour: If PowerNode machine is still up """ ocpversion = get_ocp_version("-") for node in powernode_machines: cmd = f"sudo virsh shutdown test-ocp{ocpversion}-{node.name}" result = exec_cmd(cmd) logger.info(f"Result of shutdown {result}") logger.info("Verifying node is down") ret = TimeoutSampler( timeout=timeout, sleep=3, func=self.verify_machine_is_down, node=node, ) logger.info(ret) if not ret.wait_for_func_status(result=True): raise UnexpectedBehaviour("Node {node.name} is still Running")
def create_ocs_secret(namespace): """ Function for creation of pull secret for OCS. (Mostly for ibmcloud purpose) Args: namespace (str): namespace where to create the secret """ secret_data = templating.load_yaml(constants.OCS_SECRET_YAML) docker_config_json = config.DEPLOYMENT["ocs_secret_dockerconfigjson"] secret_data["data"][".dockerconfigjson"] = docker_config_json secret_manifest = tempfile.NamedTemporaryFile( mode="w+", prefix="ocs_secret", delete=False ) templating.dump_data_to_temp_yaml(secret_data, secret_manifest.name) exec_cmd(f"oc apply -f {secret_manifest.name} -n {namespace}", timeout=2400)
def login(self, user, password): """ Logs user in Args: user (str): Name of user to be logged in password (str): Password of user to be logged in Returns: str: output of login command """ command = ["oc", "login", "-u", user, "-p", password] status = exec_cmd(command, secrets=[password]) # if on Proxy environment and if ENV_DATA["client_http_proxy"] is # defined, update kubeconfig file with proxy-url parameter to redirect # client access through proxy server if config.DEPLOYMENT.get("proxy") and config.ENV_DATA.get( "client_http_proxy"): kubeconfig = os.getenv("KUBECONFIG") if not kubeconfig or not os.path.exists(kubeconfig): kubeconfig = os.path.join( config.ENV_DATA["cluster_path"], config.RUN.get("kubeconfig_location"), ) update_kubeconfig_with_proxy_url_for_client(kubeconfig) return status
def exec_mcg_cmd(self, cmd, namespace=None, **kwargs): """ Executes an MCG CLI command through the noobaa-operator pod's CLI binary Args: cmd (str): The command to run namespace (str): The namespace to run the command in Returns: str: stdout of the command """ kubeconfig = os.getenv('KUBECONFIG') if kubeconfig: kubeconfig = f"--kubeconfig {kubeconfig} " namespace = f'-n {namespace}' if namespace else f'-n {self.namespace}' result = exec_cmd( f'oc {kubeconfig} {namespace} rsh {self.operator_pod.name} ' f'{constants.NOOBAA_OPERATOR_POD_CLI_PATH} {cmd} {namespace}', **kwargs) result.stdout = result.stdout.decode() result.stderr = result.stderr.decode() return result
def stop(self, node, timeout): """ Stop the given service using systemctl. Args: node (object): Node objects timeout (int): time in seconds to wait for service to stop. Raises: UnexpectedBehaviour: If service on PowerNode machine is still up """ nodeip = self.nodes[node.name] cmd = ( f"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@{self.bastion_ip} ssh core@{nodeip} " f"sudo systemctl stop {self.service_name}.service") if self.force: cmd += " -f" result = exec_cmd(cmd) logger.info( f"Result of shutdown {result}. Checking if service {self.service_name} went down." ) ret = TimeoutSampler( timeout=timeout, sleep=3, func=self.verify_service, node=node, action=INACTIVE, ) if not ret.wait_for_func_status(result=True): raise UnexpectedBehaviour( f"Service {self.service_name} on Node {node.name} is still Running" )
def restart(self, node, timeout): """ Restart the given service using systemctl. Args: node (object): Node objects timeout (int): time in seconds to wait for service to be started. """ nodeip = self.nodes[node.name] cmd = ( f"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@{self.bastion_ip} ssh core@{nodeip} " "sudo systemctl restart {self.service_name}.service" ) result = exec_cmd(cmd) ret = TimeoutSampler( timeout=timeout, sleep=3, func=self.verify_service, node=node, action=ACTIVE, ) logger.info( f"Result of restart of service {self.service_name} is {result}-{ret}" )
def verify_service(self, node, action): """ Verify if PowerNode is completely powered off Args: node (object): Node objects action (string): ACTIVE or INACTIVE or FAILED Returns: bool: True if service state is reqested action, False otherwise """ nodeip = self.nodes[node.name] result = exec_cmd( f"ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null root@{self.bastion_ip} ssh core@{nodeip} " f"sudo systemctl is-active {self.service_name}.service", ignore_error=True, ) output = result.stdout.lower().rstrip() if INACTIVE in output: output = INACTIVE elif ACTIVE in output: output = ACTIVE elif FAILED in output: output = FAILED if output == action: logger.info("Action succeeded.") return True else: logger.info("Action pending.") return False
def generate_onboarding_token(): """ Generate Onboarding token for consumer cluster via following steps: 1. Download ticketgen.sh script from: https://raw.githubusercontent.com/jarrpa/ocs-operator/ticketgen/hack/ticketgen/ticketgen.sh 2. Save private key from AUTH["managed_service"]["private_key"] to temporary file. 3. Run ticketgen.sh script to generate Onboarding token. Raises: CommandFailed: In case the script ticketgen.sh fails. ConfigurationError: when AUTH["managed_service"]["private_key"] not is not defined Returns: string: Onboarding token """ logger.debug("Generate onboarding token for ODF to ODF deployment") ticketgen_script_path = os.path.join(constants.DATA_DIR, "ticketgen.sh") # download ticketgen.sh script logger.debug("Download and prepare ticketgen.sh script") download_file( "https://raw.githubusercontent.com/jarrpa/ocs-operator/ticketgen/hack/ticketgen/ticketgen.sh", ticketgen_script_path, ) # add execute permission to the ticketgen.sh script current_file_permissions = os.stat(ticketgen_script_path) os.chmod( ticketgen_script_path, current_file_permissions.st_mode | stat.S_IEXEC, ) # save private key to temp file logger.debug("Prepare temporary file with private key") private_key = config.AUTH.get("managed_service", {}).get("private_key", "") if not private_key: raise ConfigurationError( "Private key for Managed Service not defined.\n" "Expected following configuration in auth.yaml file:\n" "managed_service:\n" ' private_key: "..."\n' ' public_key: "..."' ) with NamedTemporaryFile( mode="w", prefix="private", suffix=".pem", delete=True ) as key_file: key_file.write(private_key) key_file.flush() logger.debug("Generate Onboarding token") ticketgen_result = exec_cmd(f"{ticketgen_script_path} {key_file.name}") ticketgen_output = ticketgen_result.stdout.decode() if ticketgen_result.stderr: raise CommandFailed( f"Script ticketgen.sh failed to generate Onboarding token:\n" f"command: '{' '.join(ticketgen_result.args)}'\n" f"stderr: {ticketgen_result.stderr.decode()}\n" f"stdout: {ticketgen_output}" ) return ticketgen_output
def add_htpasswd_user(username, password, htpasswd_path): """ Create a new user credentials with provided username and password. These will be saved in file located on htpasswd_path. The file will be created if it doesn't exist. Args: username (str): Name of a new user password (str): Password for a new user htpasswd_path (str): Path to httpasswd file """ if os.path.isfile(htpasswd_path): cmd = ['htpasswd', '-B', '-b', htpasswd_path, username, password] else: cmd = ['htpasswd', '-c', '-B', '-b', htpasswd_path, username, password] exec_cmd(cmd, secrets=[password])