def delete_namespace_rolebinding(
name: str,
namespace: str,
) -> client.V1Status:
"""
Delete a role binding in a namespace
Args:
name: role binding name
namespace: namespace name
Returns:
V1Status object
Raises:
ApiException if something wrong with binding deletion
"""
body = client.V1DeleteOptions()
exception_count = 3
while True:
try:
api_response = client. \
RbacAuthorizationV1Api(). \
delete_namespaced_role_binding(name, namespace, body)
except ApiException as e_obj:
utilprocs.log(
("Exception when calling RbacAuthorizationV1Api->"
"delete_namespaced_role_binding: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def find_namespace(self, namespace):
"""
Finds the namespace of the particular environment
cluster in the pod.
:param namespace: namespace.
:return: A dict containing the output and the exit code\
of the command run on the test pod
"""
exception_count = 3
while True:
try:
v1_namespace_list = self.core_v1.list_namespace()
except ApiException as e_obj:
utilprocs.log(("Exception when trying to find "
"namespace: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
i = []
for i in v1_namespace_list.items:
if i.metadata.name == namespace:
break
return i
def delete_namespace(
self,
namespace: str,
) -> client.V1Status:
"""
Delete a namespace
Args:
namespace: namespace name
Returns:
V1Status object
Raises:
ApiException if something wrong with binding creation
"""
body = client.V1DeleteOptions()
exception_count = 3
while True:
try:
api_response = self.core_v1.delete_namespace(
namespace, body=body, grace_period_seconds=0)
except ApiException as e_obj:
utilprocs.log(("Exception when calling CoreV1Api->"
"delete_namespace: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def wait_for_all_pods_to_terminate(self,
namespace,
exclude_pods_list=None,
counter=60):
"""
Waits until all pods in the namespace have
been terminated before returning. Pods which
should be excluded from the termination check
are listed in exclude_pod_list.To exclude
the framework test pod, self._test_pod_prefix should
be added to exclude_pods_list.
:param namespace: namespace.
:param exclude_pods_list: pod names to be excluded
:param counter: counter for timer with interval of 10's
"""
all_pod_names = []
if exclude_pods_list is None:
exclude_pods_list = []
if self._test_pod_prefix in exclude_pods_list:
api_response = self.list_pods_from_namespace(namespace)
items = api_response.items
# all pod names list
all_pod_names += (item.metadata.name for item in items)
# regex for matching only the test-pod
regex = re.compile('^{}*'.format(self._test_pod_prefix))
# list with only the test_pod
test_pod = list(filter(regex.match, all_pod_names))
# remove the test pod string from the exclude pod list
exclude_pods_list.remove(self._test_pod_prefix)
# Add the full test pod to the exclude pod list
exclude_pods_list.extend(test_pod)
while True:
api_response = self.list_pods_from_namespace(namespace)
items = api_response.items
pod_to_be_checked = []
# list containing only those pod/pods to be checked
pod_to_be_checked += (
item.metadata.name for item in items
if item.metadata.name not in exclude_pods_list)
if not pod_to_be_checked:
break
# will check only those pods not in the excluded list
utilprocs.log('\n'.join([
'\nPhase: {} Podname: {}'.format(item.status.phase,
item.metadata.name)
for item in items if item.metadata.name in pod_to_be_checked
]))
if counter > 0:
counter = counter - 1
time.sleep(10)
else:
raise ValueError('Timeout waiting for pods to terminate')
def get_service_cluster_ip(self, service, namespace):
"""
Obtains the cluster ip of a service.
:param service: service name.
:param namespace: namespace.
"""
utilprocs.log('Getting service {} Cluster IP'.format(service))
cluster_ip = ''
api_response = self.get_namespaced_service_details(service, namespace)
items = api_response.items
for item in items:
if service in item.metadata.name:
cluster_ip = item.spec.cluster_ip
utilprocs.log('Cluster ip of the service {}: {}'.format(
service, cluster_ip))
if cluster_ip == "":
raise RuntimeError(
'No cluster ip found for service {}'.format(service))
return cluster_ip
def create_namespace(self, namespace: str) -> client.V1Namespace:
"""
Create a new namespace. It could be use to run the test
pod and the relevant micro-services
Args:
namespace: name of the namespace to create
Returns:
client.V1Namespace object
Raises:
ApiException if something wrong with namespace creation
"""
body = client.V1Namespace()
body.metadata = client.V1ObjectMeta(name=namespace)
exception_count = 3
while True:
try:
api_response = self.core_v1.create_namespace(body)
except ApiException as e_obj:
utilprocs.log(
("Exception when calling CoreV1Api->create_namespace"
": {}").format(e_obj))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def create_namespace_rolebinding(self, namespace: str,
metadata: client.V1ObjectMeta,
role_ref: client.V1RoleRef,
subjects: List[client.V1Subject]):
"""
Create a role binding in a namespace
Args:
namespace: namespace name
metadata: metadata for role binding
role_ref: Role reference for the binding
subjects: list of subjects for the binding
Returns:
V1ClusterRoleBinding object
Raises:
ApiException if something wrong with binding creation
Examples:
create_rolebinding(
metadata = client.V1ObjectMeta(
name="role-bind-name",
namespace='namespace-name'
),
role_ref = client.V1RoleRef(
api_group='rbac.authorization.k8s.io',
kind='ClusterRole',
name='cluster-admin'
),
subjects = [
client.V1Subject(
kind='ServiceAccount',
name='default',
namespace='namespace-name'
)
]
"""
body = client.V1ClusterRoleBinding(metadata=metadata,
role_ref=role_ref,
subjects=subjects)
exception_count = 3
while True:
try:
api_response = self.rbac_v1. \
create_namespaced_role_binding(namespace, body)
except ApiException as e_obj:
utilprocs.log(
("Exception when calling RbacAuthorizationV1Api->"
"create_cluster_role_binding: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def shoot_pod(self, name, namespace):
"""
Deletes the pod.
:param self: self is an instance of the class.
:param name: name of the pod to be deleted.
:param namespace: namespace.
"""
utilprocs.log('This function is deprecated. ' 'Use delete_pod instead')
self.delete_pod(name, namespace, wait_for_terminating=False)
def get_pod_ip(self, name):
"""
Obtains the ip address of the pod.
:param self: self is an instance of the class.
:param name: pod name.
"""
utilprocs.log('This function is deprecated. '
'Please use the get_pod_ip_address')
utilprocs.log('Getting pod {} IP'.format(name))
pod_addr = ''
api_response = self.get_namespace_pod_details(name)
exception_count = 3
while True:
try:
api_response = self.get_namespace_pod_details(name)
except Exception as e_obj:
utilprocs.log(("Exception when trying obtain ip "
"address of pod: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
items = api_response.items
for item in items:
if name in item.metadata.name:
pod_addr = item.status.pod_ip
utilprocs.log('ip: {}'.format(pod_addr))
return pod_addr
def create_namespace_statefulset(
self, namespace: str, metadata: client.V1ObjectMeta,
spec: client.V1StatefulSetSpec) -> client.V1Status:
"""
Create a statefulset in a namespace
Args:
:param namespace: namespace name
:param metadata: metadata for statefulset
:param spec: desired identities of pods
Returns:
V1Status object
Raises:
ApiException if something wrong with statefulset creation
Examples:
create_statefulset(
namespace='namespace-name',
metadata = client.V1ObjectMeta(
name="statefulset-name",
),
spec = client.V1StatefulSetSpec(
service_name='statefulset-name',
replicas=1,
client.V1PodTemplateSpec(
..
)
)
)
"""
body = client.V1StatefulSet(metadata=metadata, spec=spec)
exception_count = 3
while True:
try:
api_response = self.apps_v1. \
create_namespaced_stateful_set(namespace, body)
except ApiException as e_obj:
utilprocs.log(
("Exception when calling " +
"create_namespaced_stateful_set->"
"create_namespaced_stateful_set: {} - {}".format(
e_obj, e_obj.reason)))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def wait_for_all_pods_to_start(self, namespace):
"""
Waits until all the pods in the cluster are ready.
:param self: self is an instance of the class.
:param namespace: namespace.
"""
def format_containers(i):
"""
Formats the container names & lists them out in a readable format.
:param i: the particular value of iteration in the loop.
"""
return '\n'.join([
'\n Containername: {}'
'\n Ready: {}'
'\n Waiting: {}'.format(
c.name, c.ready,
str(c.state.waiting).replace('\n', ''))
for c in i.status.container_statuses
])
utilprocs.log('Pods:')
counter = 60
while True:
api_response = self.list_pods_from_namespace(namespace)
if all(i.status.container_statuses for i in api_response.items):
utilprocs.log('\n'.join([
'\nPodname: {}'
'\n Phase: {}'
'\n Containers: {}'.format(i.metadata.name,
i.status.phase,
format_containers(i))
for i in api_response.items
]))
if all([
i.status.phase == 'Running' and all(
[cs.ready for cs in i.status.container_statuses])
for i in api_response.items
]):
return
if counter > 0:
counter = counter - 1
time.sleep(10)
else:
raise ValueError('Timeout waiting for pods to reach '
'Ready & Running')
def create_namespace_secret(self, name: str, namespace: str,
secret_type: str,
data: dict) -> client.V1Secret:
"""
Create a secret in a namespace
Args:
name: secret name
namespace: namespace name
secret_type: secret type
data: data to save in the secret
Returns:
client.V1Secret object
Raises:
ApiException if something wrong with secret creation
Examples:
create_namespace_secret(
namespace='namespace-name',
name='rbd-client-secret',
secret_type='kubernetes.io/rbd',
data={'key': 'base64-encrypted-string'}
)
"""
body = client.V1Secret(
type=secret_type,
metadata=client.V1ObjectMeta(namespace=namespace, name=name),
data=data,
)
exception_count = 3
while True:
try:
api_response = self.core_v1.create_namespaced_secret(
namespace, body)
except ApiException as e_obj:
utilprocs.log(("Exception when calling CoreV1Api->"
"create_namespaced_secret: {}").format(e_obj))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def wait_for_pod_status(self,
name,
namespace,
counter=60,
ready=True,
interval=10):
"""
Waits till all the containers of the pod are ready,
or all are not ready.
:param self: self is an instance of the class.
:param name: name of the pod we are checking.
:param namespace: The namespace that the pod is in.
:param counter: counter for timer with interval of [interval]'s
:param ready: if True (default) wait for container ready,
otherwise wait for container to fail readiness probe
:param interval: interval to wait between checks in seconds, default 10
"""
utilprocs.log('Waiting for Pod: {}'.format(name))
exception_count = 5
while True:
try:
api_response = \
self.core_v1.read_namespaced_pod(name, namespace)
except Exception as e_obj:
utilprocs.log(("Exception when trying to find "
"pod in namespace: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
time.sleep(1)
continue
if ready:
if api_response.status.phase == 'Running' \
and all(container_status.ready for
container_status in
api_response.status.container_statuses):
utilprocs.log('Pod ready: {}'.format(name))
return
else:
if api_response.status.phase == 'Running' \
and all(not container_status.ready for
container_status in
api_response.status.container_statuses):
utilprocs.log('Pod not ready: {}'.format(name))
return
if counter > 0:
counter = counter - 1
time.sleep(interval)
else:
raise ValueError('Timeout waiting for pod to reach '
'Ready & Running')
def _retrieve_pod_logs(self, pod: V1Pod, scenario: str, namespace: str,
logpath: str):
"""
Retrieve logs from a pod
Args:
pod: pod object as returned from self.core_v1.list_namespaced_pod
scenario: string that you want to add to log
file names. It could be:
- <string>: a string that is meaningful to you
- 'caller': the name of the caller function will be
appended to the file names
namespace: namespace name
logpath: path where to save the logs
Returns:
True if successful, False otherwise
"""
for container in pod.spec.containers:
if scenario != '':
logname = '{}{}_{}_{}.log'.format(logpath, pod.metadata.name,
container.name, scenario)
else:
logname = '{}{}_{}.log'.format(logpath, pod.metadata.name,
container.name)
try:
log = self.core_v1.read_namespaced_pod_log(
name=pod.metadata.name,
namespace=namespace,
container=container.name)
except Exception as e_obj:
utilprocs.log(
'Error while retrieving logs from '
'container {} in pod {} in namespace {}: {}'.format(
container.name, pod.metadata.name, namespace, e_obj))
continue
if not self._write_log(logname, log):
return False
return True
def _write_log(logname: str, log: str) -> bool:
"""
Write k8s pod log to file
Args:
logname: destination log file name
log: pod log as returned from read_namespaced_pod_log
Returns:
True if successful, False otherwise
"""
try:
with open(logname, 'w+') as f_log:
f_log.write('{}\n'.format(log))
except Exception as e_obj:
utilprocs.log('Error writing file {}: {}'.format(logname, e_obj))
return False
return True
def delete_pod(self,
name,
namespace,
counter=60,
wait_for_terminating=True):
"""
Deletes the pod and by default waits for the pod to reach
terminating state.
:param self: self is an instance of the class.
:param name: name of the pod to be deleted.
:param namespace: namespace.
:param counter: counter for timer with interval of 10's.
:param wait_for_terminating: if true (default) waits for pod to reach
terminating state, if False only runs the API call
"""
utilprocs.log('Deleting pod: {}'.format(name))
exception_count = 3
while True:
try:
self.core_v1.delete_namespaced_pod(
name,
namespace,
grace_period_seconds=0,
body=client.V1DeleteOptions())
except ApiException as e_obj:
utilprocs.log(("Exception when trying to delete "
"pod in namespace: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
if wait_for_terminating:
self.wait_for_pod_status(name,
namespace,
counter,
ready=False,
interval=2)
return
def wait_for_all_replica_set(self, namespace):
"""
Waits until the Replica sets are ready.
:param self: self is an instance of the class.
:param namespace: namespace.
"""
exception_count = 3
while True:
try:
api_response = \
self.apps_v1.list_namespaced_replica_set(namespace)
except Exception as e_obj:
utilprocs.log(("Exception when trying to find "
"replica set in namespace: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
utilprocs.log('Replica sets:')
utilprocs.log([(i.metadata.name,
'Replicas ready/desired: ({}/{})'.format(
str(i.status.ready_replicas), i.spec.replicas))
for i in api_response.items])
return
def exec_cmd_on_pod(self, name, namespace, command, container=''):
"""
Executes commands on a given container of the pod.
:param self: self is an instance of the class.
:param name: name of the test pod.
:param namespace: namespace.
:param command: the command you wish to execute.
:param container: the container you want to use.
"""
exception_count = 3
while True:
try:
api_response = self.core_v1.connect_get_namespaced_pod_exec
except Exception as e_obj:
utilprocs.log(("Exception when trying to connect GET "
"requests to exec of Pod: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
utilprocs.log('Executing {} on pod {}'.format(command, name))
res = stream(api_response,
name,
namespace,
command=command,
container=container,
stderr=True,
stdin=False,
stdout=True,
tty=False)
utilprocs.log('Result: {}'.format(res))
return res
def delete_namespace_secret(self, namespace: str,
name: str) -> client.V1Status:
"""
Delete a secret from a namespace
Args:
namespace: namespace name
name: secret name
Returns:
V1Status object
Raises:
ApiException if something wrong with secret deletion
"""
body = client.V1DeleteOptions()
exception_count = 3
while True:
try:
api_response = self.core_v1.delete_namespaced_secret(
name, namespace, body=body, grace_period_seconds=0)
except ApiException as e_obj:
try:
utilprocs.log(
("Exception when calling CoreV1Api->"
"delete_namespaced_secret: {}".format(e_obj)))
except Exception:
# The above statement could fail if this method is used
# from bootstrap.py, because it runs outside the test pod
# and /var/log (default directory for utilprocs.log) is not
# writable
print(("Exception when calling CoreV1Api->"
"delete_namespaced_secret: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def get_namespace_pod_details(self, namespace):
"""
Listing all pod details in a specific namespace.
:rtype: object
:param self: self is an instance of the class.
:param namespace: namespace.
"""
exception_count = 3
while True:
try:
api_response = self.core_v1.list_namespaced_pod(
namespace, label_selector="app, release")
except Exception as e_obj:
utilprocs.log(("Exception when trying to list all "
"pod details in namespace: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def get_pod_logs(self,
namespace: str,
name: str = 'all',
scenario: str = '') -> bool:
"""
Retrieve k8s logs for the pod in the namespace
Args:
namespace: namespace name
name: name of the pod you want the log for. 'all' will retrieve
logs from all the pods in the namespace. Default to 'all'
scenario: string that you want to add to log
file names. It could be:
- <string>: a string that is meaningful to you
- 'caller': the name of the caller function will be
appended to the file names
Returns:
True if succeed, False otherwise
"""
logpath = "/var/log/"
if scenario == 'caller':
try:
# get the name of the caller function
scenario = inspect.stack()[1][3]
except Exception as e_obj:
utilprocs.log(
'Error while retrieving caller function name: {}'.format(
e_obj))
return False
pods = []
if name != 'all':
podlist = self.list_pods_from_namespace(namespace, name)
if len(podlist.items) > 1:
utilprocs.log('Too many result for pod name {}'.format(name))
return False
if not podlist.items:
utilprocs.log('No pod found with name {}'.format(name))
return False
for pod in podlist.items:
pods.append(pod)
else:
podlist = self.list_pods_from_namespace(namespace)
for pod in podlist.items:
podname = pod.metadata.name
if podname.startswith(self._test_pod_prefix):
continue
pods.append(pod)
for pod in pods:
if not self._retrieve_pod_logs(pod, scenario, namespace, logpath):
return False
return True
def delete_namespace_statefulset(
name: str,
namespace: str,
) -> client.V1Status:
"""
Delete a statefulset in a namespace
Args:
name: statefulset name
namespace: namespace name
Returns:
V1Status object
Raises:
ApiException if something wrong with statefulset deletion
"""
body = client.V1DeleteOptions()
exception_count = 3
while True:
try:
api_response = client.AppsV1Api(). \
delete_namespaced_stateful_set(
name=name,
namespace=namespace,
body=body)
except ApiException as e_obj:
utilprocs.log(
("Exception when calling AppsV1Api->"
"delete_namespaced_stateful_set: {} - {}".format(
e_obj, e_obj.reason)))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def list_statefulset_from_namespace(self, namespace, name=None) -> list:
"""
Retrieves list of statefulset from the namespace. If a specific
statefulset name is specified, the method will retrieve only
that statefulset from the namespace
Args:
namespace: namespace name
name: name of the statefulset
Returns:
List of statefulset found
Raises:
ApiException if any error during api call
"""
if name is None:
field_selector = ""
else:
field_selector = 'metadata.name={}'.format(name)
statefulset_list = []
exception_count = 3
while True:
try:
statefulset_list = self.apps_v1.list_namespaced_stateful_set(
namespace, field_selector=field_selector)
break
except Exception as e_obj:
utilprocs.log(
("Exception when trying to find statefulsets"
"in namespace:{} Error: {}".format(namespace, e_obj)))
exception_count -= 1
if exception_count <= 0:
raise ApiException
continue
return statefulset_list.items
def get_namespace_secrets(
self,
namespace: str,
secret_names: List[str] = None) -> List[client.V1Secret]:
"""
Retrieve secrets from a namespace
Args:
namespace: namespace name
secret_names: list of all the secrets to retrieve.
defaults to None
Returns:
list of secrets found
Raises:
ApiException if any error during api call
"""
exception_count = 3
while True:
try:
api_response = self.core_v1.list_namespaced_secret(namespace)
except ApiException as e_obj:
utilprocs.log(("Exception when calling CoreV1Api->"
"list_namespaced_secret: {}").format(e_obj))
exception_count -= 1
if exception_count <= 0:
raise
continue
secrets = []
for secret in api_response.items:
if (secret_names is not None
and secret.metadata.name in secret_names):
secrets.append(secret)
return secrets
def list_pods_from_namespace(self, namespace, name=None):
"""
Retrieves list of pods from the namespace
Args:
namespace: namespace name
name: name of the pod.
Returns:
List of pods found
Raises:
ApiException if any error during api call
"""
if name is None:
field_selector = ""
else:
field_selector = 'metadata.name={}'.format(name)
podlist = []
exception_count = 3
while True:
try:
podlist = self.core_v1.list_namespaced_pod(
namespace, field_selector=field_selector)
break
except Exception as e_obj:
utilprocs.log(
("Exception when trying to find pods"
"in namespace:{} Error: {}".format(namespace, e_obj)))
exception_count -= 1
if exception_count <= 0:
raise ApiException
continue
return podlist
def get_namespaced_service_details(self, service, namespace):
"""
Listing details of a service in a specific namespace.
:param service: service name.
:param namespace: namespace.
"""
exception_count = 3
while True:
try:
utilprocs.log('Getting details of {} service'.format(service))
api_response = self.core_v1.list_namespaced_service(
namespace,
field_selector="metadata.name={}".format(service))
except Exception as e_obj:
utilprocs.log(
("Exception when trying to list details of "
"a service in namespace {}: {}".format(namespace, e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
return api_response
def delete_all_pvc_namespace(self, namespace):
"""
Deletes all Persistent Volume Claims in the specific namespace.
:param self: self is an instance of the class.
:param namespace: namespace.
"""
exception_count = 3
while True:
try:
self.core_v1. \
delete_collection_namespaced_persistent_volume_claim(
namespace)
break
except Exception as e_obj:
utilprocs.log(("Exception when trying to delete "
"all Persistent Volume Claims "
"in namespace: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
# keep it to 2 minutes so k8s has enough time to delete the pods
# because deletion of the PVCs starts after pods deletion
try_count = 24
while try_count > 0:
response = self.core_v1. \
list_namespaced_persistent_volume_claim(namespace)
if response.items:
time.sleep(5)
try_count -= 1
continue
utilprocs.log(
"All PVC's are no longer in namespace {}".format(namespace))
break
else:
utilprocs.log("Timeout waiting for PVC's to terminate, they may "
"still be in namespace: {}".format(namespace))
raise RuntimeError('Timeout waiting for PVC deletion')
def wait_for_statefulset(self,
namespace,
counter=60,
name: str = None) -> bool:
"""Waits until all replicas are ready
Args:
namespace: The namespace that the pod is in.
counter: Is the number of intervals with a
10 seconds sleep after each interval.
name: name of the statefulset we are checking.
Returns;
True if all pods in statefulset have reached the ready status
Raises:
RuntimeError if waiting timeout reached
"""
utilprocs.log('Waiting for StatefulSet {}'.format(name))
exception_count = 3
while True:
try:
statefulset_list = self.list_statefulset_from_namespace(
namespace=namespace, name=name)
except Exception as e_obj:
utilprocs.log(("Exception when trying to find "
"statefulset in namespace: {}".format(e_obj)))
exception_count -= 1
if exception_count <= 0:
raise
continue
# after retrieved the list of statefulset in the namespace
# loop through it to check if the number of replicas configured
# is equal to the number of replicas already deployed: if they're
# equal remove the statefulset from the list. When the list is
# empty, the loop exits
for idx, statefulset in enumerate(statefulset_list):
ready_replicas = statefulset.status.ready_replicas
total_replicas = statefulset.status.replicas
updated_replicas = statefulset.status.updated_replicas
if ready_replicas == total_replicas and \
updated_replicas == total_replicas:
utilprocs.log('StatefulSet ready: {}'.format(
statefulset.metadata.name))
del statefulset_list[idx]
if not statefulset_list:
utilprocs.log('All StatefulSet reached ready status')
break
utilprocs.log(
'{} StatefulSets need to reach the ready status'.format(
len(statefulset_list)))
counter -= 1
if counter <= 0:
raise RuntimeError(
'Timeout waiting for StatefulSet to reach ready status')
time.sleep(10)
return True
