def wait_for_running_pods_ready(pod_names=None, namespace=None, all_namespaces=False, labels=None, timeout=300, fail_ok=False, con_ssh=None, exclude=False, strict=False, **kwargs): """ Wait for Running pods to be Ready, such as 1/1, 3/3 Args: pod_names: namespace: all_namespaces: labels: timeout: fail_ok: con_ssh: exclude: strict: **kwargs: Returns (bool): """ unready_pods = get_unready_running_pods(namespace=namespace, all_namespaces=all_namespaces, pod_names=pod_names, labels=labels, exclude=exclude, strict=strict, con_ssh=con_ssh, **kwargs) if not unready_pods: return True end_time = time.time() + timeout while time.time() < end_time: pods_info = get_pods(field=('NAME', 'READY'), namespace=namespace, all_namespaces=all_namespaces, pod_names=unready_pods, con_ssh=con_ssh) for pod_info in pods_info: pod_name, pod_ready = pod_info ready_count, total_count = pod_ready.split('/') if ready_count == total_count: unready_pods.remove(pod_name) if not unready_pods: return True msg = "Some pods are not ready within {}s: {}".format( timeout, unready_pods) LOG.warning(msg) if fail_ok: return False raise exceptions.KubeError(msg)
def wait_for_resources_gone(resource_names=None, resource_type='pod', namespace=None, timeout=120, check_interval=3, con_ssh=None, fail_ok=False, strict=True, exclude=False, **kwargs): """ Wait for pod(s) to be gone from kubectl get Args: resource_names (str|list|tuple): full name of a pod resource_type (str): namespace (None|str): timeout: check_interval: con_ssh: fail_ok: strict (bool): exclude **kwargs Returns (tuple): (True, None) (False, <actual_pods_info>) # actual_pods_info is a dict with pod_name as key, and pod_info(dict) as value """ end_time = time.time() + timeout resources_to_check = resource_names while time.time() < end_time: resources_to_check = get_resources(resource_names=resources_to_check, namespace=namespace, resource_type=resource_type, con_ssh=con_ssh, fail_ok=True, strict=strict, exclude=exclude, **kwargs) if not resources_to_check: return True, resources_to_check time.sleep(check_interval) msg = 'Resources did not disappear in {} seconds. Remaining resources: ' \ '{}, namespace: {}'.format(timeout, resources_to_check, namespace) if fail_ok: LOG.info(msg) return False, resources_to_check raise exceptions.KubeError(msg)
def wait_for_nodes_ready(hosts=None, timeout=120, check_interval=5, con_ssh=None, fail_ok=False): """ Wait for hosts in ready state via kubectl get nodes Args: hosts (None|list|str|tuple): Wait for all hosts ready if None is specified timeout: check_interval: con_ssh: fail_ok: Returns (tuple): (True, None) (False, <nodes_not_ready>(list)) """ if hosts and isinstance(hosts, str): hosts = [hosts] end_time = time.time() + timeout nodes_not_ready = None while time.time() < end_time: nodes_not_ready = get_nodes(status='Ready', field='NAME', exclude=True, con_ssh=con_ssh, fail_ok=True) if nodes_not_ready and hosts: nodes_not_ready = list(set(nodes_not_ready) & set(hosts)) if nodes_not_ready: LOG.info('{} not ready yet'.format(nodes_not_ready)) elif nodes_not_ready is not None: LOG.info("All nodes are ready{}".format( ': {}'.format(hosts) if hosts else '')) return True, None time.sleep(check_interval) msg = '{} are not ready within {}s'.format(nodes_not_ready, timeout) LOG.warning(msg) if fail_ok: return False, nodes_not_ready else: raise exceptions.KubeError(msg)
def get_pod_logs(pod_name, namespace='openstack', grep_pattern=None, tail_count=10, strict=False, fail_ok=False, con_ssh=None): """ Get logs for given pod via kubectl logs cmd Args: pod_name (str): partial or full pod_name. If full name, set strict to True. namespace (str|None): grep_pattern (str|None): tail_count (int|None): strict (bool): fail_ok: con_ssh: Returns (str): """ if pod_name and not strict: grep = '-E -i "{}|NAME"'.format(pod_name) pod_name = get_resources(namespace='openstack', resource_type='pod', con_ssh=con_ssh, rtn_list=True, grep=grep, fail_ok=fail_ok)[0].get('name') namespace = '-n {} '.format(namespace) if namespace else '' grep = '' if grep_pattern: if isinstance(grep_pattern, str): grep_pattern = (grep_pattern, ) grep = ''.join([ ' | grep --color=never {}'.format(grep_str) for grep_str in grep_pattern ]) tail = ' | tail -n {}'.format(tail_count) if tail_count else '' args = '{}{}{}{}'.format(namespace, pod_name, grep, tail) code, output = exec_kube_cmd(sub_cmd='logs', args=args, con_ssh=con_ssh) if not output and not fail_ok: raise exceptions.KubeError( "No kubectl logs found with args: {}".format(args)) return output
def wait_for_pods_healthy(pod_names=None, namespace=None, all_namespaces=True, labels=None, timeout=300, check_interval=5, con_ssh=None, fail_ok=False, exclude=False, strict=False, **kwargs): """ Wait for pods ready Args: pod_names (list|tuple|str|None): full name of pod(s) namespace (str|None): all_namespaces (bool|None) labels (str|dict|list|tuple|None): timeout: check_interval: con_ssh: fail_ok: exclude (bool) strict (bool): strict applies to node and name matching if given **kwargs Returns (tuple): """ LOG.info("Wait for pods ready..") if not pod_names: pod_names = None elif isinstance(pod_names, str): pod_names = [pod_names] bad_pods = None end_time = time.time() + timeout while time.time() < end_time: bad_pods_info = get_unhealthy_pods(labels=labels, field=('NAME', 'STATUS'), namespace=namespace, all_namespaces=all_namespaces, con_ssh=con_ssh, exclude=exclude, strict=strict, **kwargs) bad_pods = { pod_info[0]: pod_info[1] for pod_info in bad_pods_info if (not pod_names or pod_info[0] in pod_names) } if not bad_pods: LOG.info("Pods are Completed or Running.") if pod_names: pod_names = [ pod for pod in pod_names if not re.search('audit-|init-', pod) ] if not pod_names: return True is_ready = wait_for_running_pods_ready( pod_names=pod_names, namespace=namespace, all_namespaces=all_namespaces, labels=labels, timeout=int(end_time - time.time()), strict=strict, con_ssh=con_ssh, fail_ok=fail_ok, **kwargs) return is_ready time.sleep(check_interval) msg = 'Some pods are not Running or Completed: {}'.format(bad_pods) LOG.warning(msg) if fail_ok: return False dump_pods_info(con_ssh=con_ssh) raise exceptions.KubeError(msg)
def wait_for_pods_status(pod_names=None, partial_names=None, labels=None, namespace=None, status=PodStatus.RUNNING, timeout=120, check_interval=3, con_ssh=None, fail_ok=False, strict=False, **kwargs): """ Wait for pod(s) to reach given status via kubectl get pod Args: pod_names (str|list|tuple): full name of the pods partial_names (str|list|tuple): Used only if pod_names are not provided labels (str|list|tuple|dict|None): Used only if pod_names are not provided namespace (None|str): status (str|None|list): None means any state as long as pod exists. timeout: check_interval: con_ssh: fail_ok: strict (bool): Returns (tuple): (True, <actual_pods_info>) # actual_pods_info is a dict with pod_name as key, and pod_info(dict) as value (False, <actual_pods_info>) """ pods_to_check = [] if pod_names: if isinstance(pod_names, str): pod_names = [pod_names] else: pod_names = list(pod_names) labels = partial_names = None pods_to_check = list(pod_names) elif partial_names: if isinstance(partial_names, str): partial_names = [partial_names] else: partial_names = list(partial_names) kwargs['NAME'] = partial_names pods_to_check = list(partial_names) actual_status = {} end_time = time.time() + timeout while time.time() < end_time: pod_full_names = pods_to_check if pod_names else None pods_values = get_pods(pod_names=pod_full_names, field=('NAME', 'status'), namespace=namespace, labels=labels, strict=strict, fail_ok=True, con_ssh=con_ssh, **kwargs) if not pods_values: # No pods returned, continue to check. time.sleep(check_interval) continue continue_check = False # This is used when only labels are provided for pod_info in pods_values: pod_name, pod_status = pod_info actual_status[pod_name] = pod_status if status and pod_status not in status: # Status not as expected, continue to wait continue_check = True if partial_names: # In this case, there might be multiple pods that matches # 1 partial name, so the partial name that # matches current pod could have been removed if there # was one other pod that also matched the name # had reached the desired state. In this case, we will # add the partial name back to check list for partial_name in partial_names: if partial_name in pod_name and partial_name not in \ pods_to_check: pods_to_check.append(partial_name) break else: # Criteria met for current pod, remove it from check_list if pod_names: pods_to_check.remove(pod_name) elif partial_names: for partial_name in partial_names: if partial_name in pod_name and partial_name in \ pods_to_check: pods_to_check.remove(partial_name) break if not pods_to_check and not continue_check: return True, actual_status time.sleep(check_interval) name_str = 'Names: {}'.format(pods_to_check) if pods_to_check else '' label_str = 'Labels: {}'.format(labels) if labels else '' criteria = '{} {}'.format(name_str, label_str).strip() msg = "Pods did not reach expected status within {}s. Criteria not met: " \ "{}. Actual info: {}".format(timeout, criteria, actual_status) if fail_ok: LOG.info(msg) return False, actual_status raise exceptions.KubeError(msg)
def get_openstack_pods(field='Name', namespace='openstack', application=None, component=None, pod_names=None, extra_labels=None, field_selectors=None, exclude_label=False, fail_ok=False, con_ssh=None, strict=True, exclude=False, **kwargs): """ Get openstack pods via kubectl get pods Note that pod labels can be found via kubectl get pods -n <namespace> --show-labels Args: field (str|list|tuple): namespace: application (str|None): label: application component (str|None): label: component pod_names extra_labels (str|None): field_selectors (str|list|tuple|dict|None): exclude_label fail_ok: con_ssh: exclude: strict: **kwargs: Returns (list): """ if pod_names: labels = None else: connector = '!=' if exclude_label else '=' labels = [] if application: labels.append('application{}{}'.format(connector, application)) if component: labels.append('component{}{}'.format(connector, component)) if extra_labels: labels.append(extra_labels) labels = ','.join(labels) pods = get_pods(pod_names=pod_names, field=field, namespace=namespace, labels=labels, fail_ok=fail_ok, field_selectors=field_selectors, strict=strict, exclude=exclude, con_ssh=con_ssh, **kwargs) if not pods: msg = "No pods found for namespace - {} with selectors: {}".format( namespace, labels) LOG.info(msg) if not fail_ok: raise exceptions.KubeError(msg) return pods