def wait_for_running_pods_ready(pod_names=None,
namespace=None,
all_namespaces=False,
labels=None,
timeout=300,
fail_ok=False,
con_ssh=None,
exclude=False,
strict=False,
**kwargs):
"""
Wait for Running pods to be Ready, such as 1/1, 3/3
Args:
pod_names:
namespace:
all_namespaces:
labels:
timeout:
fail_ok:
con_ssh:
exclude:
strict:
**kwargs:
Returns (bool):
"""
unready_pods = get_unready_running_pods(namespace=namespace,
all_namespaces=all_namespaces,
pod_names=pod_names,
labels=labels,
exclude=exclude,
strict=strict,
con_ssh=con_ssh,
**kwargs)
if not unready_pods:
return True
end_time = time.time() + timeout
while time.time() < end_time:
pods_info = get_pods(field=('NAME', 'READY'),
namespace=namespace,
all_namespaces=all_namespaces,
pod_names=unready_pods,
con_ssh=con_ssh)
for pod_info in pods_info:
pod_name, pod_ready = pod_info
ready_count, total_count = pod_ready.split('/')
if ready_count == total_count:
unready_pods.remove(pod_name)
if not unready_pods:
return True
msg = "Some pods are not ready within {}s: {}".format(
timeout, unready_pods)
LOG.warning(msg)
if fail_ok:
return False
raise exceptions.KubeError(msg)
def wait_for_resources_gone(resource_names=None,
resource_type='pod',
namespace=None,
timeout=120,
check_interval=3,
con_ssh=None,
fail_ok=False,
strict=True,
exclude=False,
**kwargs):
"""
Wait for pod(s) to be gone from kubectl get
Args:
resource_names (str|list|tuple): full name of a pod
resource_type (str):
namespace (None|str):
timeout:
check_interval:
con_ssh:
fail_ok:
strict (bool):
exclude
**kwargs
Returns (tuple):
(True, None)
(False, <actual_pods_info>) # actual_pods_info is a dict with
pod_name as key, and pod_info(dict) as value
"""
end_time = time.time() + timeout
resources_to_check = resource_names
while time.time() < end_time:
resources_to_check = get_resources(resource_names=resources_to_check,
namespace=namespace,
resource_type=resource_type,
con_ssh=con_ssh,
fail_ok=True,
strict=strict,
exclude=exclude,
**kwargs)
if not resources_to_check:
return True, resources_to_check
time.sleep(check_interval)
msg = 'Resources did not disappear in {} seconds. Remaining resources: ' \
'{}, namespace: {}'.format(timeout, resources_to_check, namespace)
if fail_ok:
LOG.info(msg)
return False, resources_to_check
raise exceptions.KubeError(msg)
def wait_for_nodes_ready(hosts=None,
timeout=120,
check_interval=5,
con_ssh=None,
fail_ok=False):
"""
Wait for hosts in ready state via kubectl get nodes
Args:
hosts (None|list|str|tuple): Wait for all hosts ready if None is
specified
timeout:
check_interval:
con_ssh:
fail_ok:
Returns (tuple):
(True, None)
(False, <nodes_not_ready>(list))
"""
if hosts and isinstance(hosts, str):
hosts = [hosts]
end_time = time.time() + timeout
nodes_not_ready = None
while time.time() < end_time:
nodes_not_ready = get_nodes(status='Ready',
field='NAME',
exclude=True,
con_ssh=con_ssh,
fail_ok=True)
if nodes_not_ready and hosts:
nodes_not_ready = list(set(nodes_not_ready) & set(hosts))
if nodes_not_ready:
LOG.info('{} not ready yet'.format(nodes_not_ready))
elif nodes_not_ready is not None:
LOG.info("All nodes are ready{}".format(
': {}'.format(hosts) if hosts else ''))
return True, None
time.sleep(check_interval)
msg = '{} are not ready within {}s'.format(nodes_not_ready, timeout)
LOG.warning(msg)
if fail_ok:
return False, nodes_not_ready
else:
raise exceptions.KubeError(msg)
def get_pod_logs(pod_name,
namespace='openstack',
grep_pattern=None,
tail_count=10,
strict=False,
fail_ok=False,
con_ssh=None):
"""
Get logs for given pod via kubectl logs cmd
Args:
pod_name (str): partial or full pod_name. If full name, set strict to
True.
namespace (str|None):
grep_pattern (str|None):
tail_count (int|None):
strict (bool):
fail_ok:
con_ssh:
Returns (str):
"""
if pod_name and not strict:
grep = '-E -i "{}|NAME"'.format(pod_name)
pod_name = get_resources(namespace='openstack',
resource_type='pod',
con_ssh=con_ssh,
rtn_list=True,
grep=grep,
fail_ok=fail_ok)[0].get('name')
namespace = '-n {} '.format(namespace) if namespace else ''
grep = ''
if grep_pattern:
if isinstance(grep_pattern, str):
grep_pattern = (grep_pattern, )
grep = ''.join([
' | grep --color=never {}'.format(grep_str)
for grep_str in grep_pattern
])
tail = ' | tail -n {}'.format(tail_count) if tail_count else ''
args = '{}{}{}{}'.format(namespace, pod_name, grep, tail)
code, output = exec_kube_cmd(sub_cmd='logs', args=args, con_ssh=con_ssh)
if not output and not fail_ok:
raise exceptions.KubeError(
"No kubectl logs found with args: {}".format(args))
return output
def wait_for_pods_healthy(pod_names=None,
namespace=None,
all_namespaces=True,
labels=None,
timeout=300,
check_interval=5,
con_ssh=None,
fail_ok=False,
exclude=False,
strict=False,
**kwargs):
"""
Wait for pods ready
Args:
pod_names (list|tuple|str|None): full name of pod(s)
namespace (str|None):
all_namespaces (bool|None)
labels (str|dict|list|tuple|None):
timeout:
check_interval:
con_ssh:
fail_ok:
exclude (bool)
strict (bool): strict applies to node and name matching if given
**kwargs
Returns (tuple):
"""
LOG.info("Wait for pods ready..")
if not pod_names:
pod_names = None
elif isinstance(pod_names, str):
pod_names = [pod_names]
bad_pods = None
end_time = time.time() + timeout
while time.time() < end_time:
bad_pods_info = get_unhealthy_pods(labels=labels,
field=('NAME', 'STATUS'),
namespace=namespace,
all_namespaces=all_namespaces,
con_ssh=con_ssh,
exclude=exclude,
strict=strict,
**kwargs)
bad_pods = {
pod_info[0]: pod_info[1]
for pod_info in bad_pods_info
if (not pod_names or pod_info[0] in pod_names)
}
if not bad_pods:
LOG.info("Pods are Completed or Running.")
if pod_names:
pod_names = [
pod for pod in pod_names
if not re.search('audit-|init-', pod)
]
if not pod_names:
return True
is_ready = wait_for_running_pods_ready(
pod_names=pod_names,
namespace=namespace,
all_namespaces=all_namespaces,
labels=labels,
timeout=int(end_time - time.time()),
strict=strict,
con_ssh=con_ssh,
fail_ok=fail_ok,
**kwargs)
return is_ready
time.sleep(check_interval)
msg = 'Some pods are not Running or Completed: {}'.format(bad_pods)
LOG.warning(msg)
if fail_ok:
return False
dump_pods_info(con_ssh=con_ssh)
raise exceptions.KubeError(msg)
def wait_for_pods_status(pod_names=None,
partial_names=None,
labels=None,
namespace=None,
status=PodStatus.RUNNING,
timeout=120,
check_interval=3,
con_ssh=None,
fail_ok=False,
strict=False,
**kwargs):
"""
Wait for pod(s) to reach given status via kubectl get pod
Args:
pod_names (str|list|tuple): full name of the pods
partial_names (str|list|tuple): Used only if pod_names are not provided
labels (str|list|tuple|dict|None): Used only if pod_names are not
provided
namespace (None|str):
status (str|None|list): None means any state as long as pod exists.
timeout:
check_interval:
con_ssh:
fail_ok:
strict (bool):
Returns (tuple):
(True, <actual_pods_info>) # actual_pods_info is a dict with
pod_name as key, and pod_info(dict) as value
(False, <actual_pods_info>)
"""
pods_to_check = []
if pod_names:
if isinstance(pod_names, str):
pod_names = [pod_names]
else:
pod_names = list(pod_names)
labels = partial_names = None
pods_to_check = list(pod_names)
elif partial_names:
if isinstance(partial_names, str):
partial_names = [partial_names]
else:
partial_names = list(partial_names)
kwargs['NAME'] = partial_names
pods_to_check = list(partial_names)
actual_status = {}
end_time = time.time() + timeout
while time.time() < end_time:
pod_full_names = pods_to_check if pod_names else None
pods_values = get_pods(pod_names=pod_full_names,
field=('NAME', 'status'),
namespace=namespace,
labels=labels,
strict=strict,
fail_ok=True,
con_ssh=con_ssh,
**kwargs)
if not pods_values:
# No pods returned, continue to check.
time.sleep(check_interval)
continue
continue_check = False # This is used when only labels are provided
for pod_info in pods_values:
pod_name, pod_status = pod_info
actual_status[pod_name] = pod_status
if status and pod_status not in status:
# Status not as expected, continue to wait
continue_check = True
if partial_names:
# In this case, there might be multiple pods that matches
# 1 partial name, so the partial name that
# matches current pod could have been removed if there
# was one other pod that also matched the name
# had reached the desired state. In this case, we will
# add the partial name back to check list
for partial_name in partial_names:
if partial_name in pod_name and partial_name not in \
pods_to_check:
pods_to_check.append(partial_name)
break
else:
# Criteria met for current pod, remove it from check_list
if pod_names:
pods_to_check.remove(pod_name)
elif partial_names:
for partial_name in partial_names:
if partial_name in pod_name and partial_name in \
pods_to_check:
pods_to_check.remove(partial_name)
break
if not pods_to_check and not continue_check:
return True, actual_status
time.sleep(check_interval)
name_str = 'Names: {}'.format(pods_to_check) if pods_to_check else ''
label_str = 'Labels: {}'.format(labels) if labels else ''
criteria = '{} {}'.format(name_str, label_str).strip()
msg = "Pods did not reach expected status within {}s. Criteria not met: " \
"{}. Actual info: {}".format(timeout, criteria, actual_status)
if fail_ok:
LOG.info(msg)
return False, actual_status
raise exceptions.KubeError(msg)
def get_openstack_pods(field='Name',
namespace='openstack',
application=None,
component=None,
pod_names=None,
extra_labels=None,
field_selectors=None,
exclude_label=False,
fail_ok=False,
con_ssh=None,
strict=True,
exclude=False,
**kwargs):
"""
Get openstack pods via kubectl get pods
Note that pod labels can be found via kubectl get pods -n <namespace>
--show-labels
Args:
field (str|list|tuple):
namespace:
application (str|None): label: application
component (str|None): label: component
pod_names
extra_labels (str|None):
field_selectors (str|list|tuple|dict|None):
exclude_label
fail_ok:
con_ssh:
exclude:
strict:
**kwargs:
Returns (list):
"""
if pod_names:
labels = None
else:
connector = '!=' if exclude_label else '='
labels = []
if application:
labels.append('application{}{}'.format(connector, application))
if component:
labels.append('component{}{}'.format(connector, component))
if extra_labels:
labels.append(extra_labels)
labels = ','.join(labels)
pods = get_pods(pod_names=pod_names,
field=field,
namespace=namespace,
labels=labels,
fail_ok=fail_ok,
field_selectors=field_selectors,
strict=strict,
exclude=exclude,
con_ssh=con_ssh,
**kwargs)
if not pods:
msg = "No pods found for namespace - {} with selectors: {}".format(
namespace, labels)
LOG.info(msg)
if not fail_ok:
raise exceptions.KubeError(msg)
return pods
