def _wait(self, deadline): ''' Waits for resources to become ready. Returns whether resources were modified, or `None` if that is to be ignored. ''' deadline_remaining = int(round(deadline - time.time())) if deadline_remaining <= 0: error = ( "Timed out waiting for resource type={}, namespace={}, " "labels={}".format( self.resource_type, self.chart_wait.release_id.namespace, self.label_selector)) LOG.error(error) raise k8s_exceptions.KubernetesWatchTimeoutException(error) timed_out, modified, unready, found_resources = ( self._watch_resource_completions(timeout=deadline_remaining)) if (not found_resources) and not self.required: return None if timed_out: if not found_resources: details = ( 'None found! Are `wait.labels` correct? Does ' '`wait.resources` need to exclude `type: {}`?'.format( self.resource_type)) else: details = ( 'These {}s were not ready={}'.format( self.resource_type, sorted(unready))) error = ( 'Timed out waiting for {}s (namespace={}, labels=({})). {}'. format( self.resource_type, self.chart_wait.release_id.namespace, self.label_selector, details)) LOG.error(error) raise k8s_exceptions.KubernetesWatchTimeoutException(error) return modified
def _watch_job_completion(self, namespace, label_selector, timeout): ''' Watch and wait for job completion. Returns when conditions are met, or raises a timeout exception. ''' try: timeout = self._check_timeout(timeout) ready_jobs = {} w = watch.Watch() for event in w.stream(self.batch_api.list_namespaced_job, namespace=namespace, label_selector=label_selector, timeout_seconds=timeout): job_name = event['object'].metadata.name LOG.debug('Watch event %s on job %s', event['type'].upper(), job_name) # Track the expected and actual number of completed pods # See: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ # noqa expected = event['object'].spec.completions completed = event['object'].status.succeeded if expected != completed: ready_jobs[job_name] = False else: ready_jobs[job_name] = True LOG.debug( 'Job %s complete (spec.completions=%s, ' 'status.succeeded=%s)', job_name, expected, completed) if all(ready_jobs.values()): return True except ApiException as e: LOG.exception( "Exception when watching jobs: namespace=%s, labels=(%s)", namespace, label_selector) raise e if not ready_jobs: LOG.warn( 'Saw no job events for namespace=%s, labels=(%s). ' 'Are the labels correct?', namespace, label_selector) return False err_msg = ('Reached timeout while waiting for job completions: ' 'namespace=%s, labels=(%s)' % (namespace, label_selector)) LOG.error(err_msg) raise exceptions.KubernetesWatchTimeoutException(err_msg)
def _delete_job_action(self, list_func, delete_func, job_type_description, name, namespace="default", propagation_policy='Foreground', timeout=DEFAULT_K8S_TIMEOUT): try: LOG.debug('Deleting %s %s, Wait timeout=%s', job_type_description, name, timeout) body = client.V1DeleteOptions() w = watch.Watch() issue_delete = True for event in w.stream(list_func, namespace=namespace, timeout_seconds=timeout): if issue_delete: delete_func(name=name, namespace=namespace, body=body, propagation_policy=propagation_policy) issue_delete = False event_type = event['type'].upper() job_name = event['object'].metadata.name if event_type == 'DELETED' and job_name == name: LOG.debug('Successfully deleted %s %s', job_type_description, job_name) return err_msg = ('Reached timeout while waiting to delete %s: ' 'name=%s, namespace=%s' % (job_type_description, name, namespace)) LOG.error(err_msg) raise exceptions.KubernetesWatchTimeoutException(err_msg) except ApiException as e: LOG.exception("Exception when deleting %s: name=%s, namespace=%s", job_type_description, name, namespace) raise e
def _delete_item_action(self, list_func, delete_func, object_type_description, name, namespace="default", propagation_policy='Foreground', timeout=DEFAULT_K8S_TIMEOUT): ''' This function takes the action to delete an object (job, cronjob, pod) from kubernetes. It will wait for the object to be fully deleted before returning to processing or timing out. :param list_func: The callback function to list the specified object type :param delete_func: The callback function to delete the specified object type :param object_type_description: The types of objects to delete, in `job`, `cronjob`, or `pod` :param name: The name of the object to delete :param namespace: The namespace of the object :param propagation_policy: The Kubernetes propagation_policy to apply to the delete. Default 'Foreground' means that child objects will be deleted before the given object is marked as deleted. See: https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#controlling-how-the-garbage-collector-deletes-dependents # noqa :param timeout: The timeout to wait for the delete to complete ''' try: timeout = self._check_timeout(timeout) LOG.debug('Watching to delete %s %s, Wait timeout=%s', object_type_description, name, timeout) body = client.V1DeleteOptions() w = watch.Watch() issue_delete = True found_events = False for event in w.stream(list_func, namespace=namespace, timeout_seconds=timeout): if issue_delete: delete_func(name=name, namespace=namespace, body=body, propagation_policy=propagation_policy) issue_delete = False event_type = event['type'].upper() item_name = event['object'].metadata.name LOG.debug('Watch event %s on %s', event_type, item_name) if item_name == name: found_events = True if event_type == 'DELETED': LOG.info('Successfully deleted %s %s', object_type_description, item_name) return if not found_events: LOG.warn('Saw no delete events for %s %s in namespace=%s', object_type_description, name, namespace) err_msg = ('Reached timeout while waiting to delete %s: ' 'name=%s, namespace=%s' % (object_type_description, name, namespace)) LOG.error(err_msg) raise exceptions.KubernetesWatchTimeoutException(err_msg) except ApiException as e: LOG.exception("Exception when deleting %s: name=%s, namespace=%s", object_type_description, name, namespace) raise e
def wait_until_ready(self, release=None, namespace='', labels='', timeout=DEFAULT_K8S_TIMEOUT, k8s_wait_attempts=1, k8s_wait_attempt_sleep=1): ''' Wait until all pods become ready given the filters provided by ``release``, ``labels`` and ``namespace``. :param release: chart release :param namespace: the namespace used to filter which pods to wait on :param labels: the labels used to filter which pods to wait on :param timeout: time before disconnecting ``Watch`` stream :param k8s_wait_attempts: The number of times to attempt waiting for pods to become ready (minimum 1). :param k8s_wait_attempt_sleep: The time in seconds to sleep between attempts (minimum 1). ''' timeout = self._check_timeout(timeout) # NOTE(MarshM) 'release' is currently unused label_selector = label_selectors(labels) if labels else '' wait_attempts = (k8s_wait_attempts if k8s_wait_attempts >= 1 else 1) sleep_time = (k8s_wait_attempt_sleep if k8s_wait_attempt_sleep >= 1 else 1) LOG.debug( "Wait on namespace=(%s) labels=(%s) for %s sec " "(k8s wait %s times, sleep %ss)", namespace, label_selector, timeout, wait_attempts, sleep_time) if not namespace: # This shouldn't be reachable LOG.warn('"namespace" not specified, waiting across all available ' 'namespaces is likely to cause unintended consequences.') if not label_selector: LOG.warn('"label_selector" not specified, waiting with no labels ' 'may cause unintended consequences.') # Track the overall deadline for timing out during waits deadline = time.time() + timeout # First, we should watch for jobs before checking pods, as a job can # still be running even after its current pods look healthy or have # been removed and are pending reschedule found_jobs = self.get_namespace_job(namespace, label_selector) if len(found_jobs.items): self._watch_job_completion(namespace, label_selector, timeout) # NOTE(mark-burnett): Attempt to wait multiple times without # modification, in case new pods appear after our watch exits. successes = 0 while successes < wait_attempts: deadline_remaining = int(round(deadline - time.time())) if deadline_remaining <= 0: LOG.info('Timed out while waiting for pods.') raise exceptions.KubernetesWatchTimeoutException( 'Timed out while waiting on namespace=(%s) labels=(%s)' % (namespace, label_selector)) timed_out, modified_pods, unready_pods, found_events = ( self._watch_pod_completions(namespace=namespace, label_selector=label_selector, timeout=deadline_remaining)) if not found_events: LOG.warn( 'Saw no install/update events for release=%s, ' 'namespace=%s, labels=(%s). Are the labels correct?', release, namespace, label_selector) if timed_out: LOG.info('Timed out waiting for pods: %s', sorted(unready_pods)) raise exceptions.KubernetesWatchTimeoutException( 'Timed out while waiting on namespace=(%s) labels=(%s)' % (namespace, label_selector)) if modified_pods: successes = 0 LOG.debug('Continuing to wait, found modified pods: %s', sorted(modified_pods)) else: successes += 1 LOG.debug('Found no modified pods this attempt. successes=%d', successes) time.sleep(sleep_time) return True
def wait_until_ready(self, release=None, namespace='', labels='', timeout=DEFAULT_K8S_TIMEOUT, k8s_wait_attempts=1, k8s_wait_attempt_sleep=1): ''' Wait until all pods become ready given the filters provided by ``release``, ``labels`` and ``namespace``. :param release: chart release :param namespace: the namespace used to filter which pods to wait on :param labels: the labels used to filter which pods to wait on :param timeout: time before disconnecting ``Watch`` stream :param k8s_wait_attempts: The number of times to attempt waiting for pods to become ready (minimum 1). :param k8s_wait_attempt_sleep: The time in seconds to sleep between attempts (minimum 1). ''' # NOTE(MarshM) 'release' is currently unused label_selector = label_selectors(labels) if labels else '' wait_attempts = (k8s_wait_attempts if k8s_wait_attempts >= 1 else 1) sleep_time = (k8s_wait_attempt_sleep if k8s_wait_attempt_sleep >= 1 else 1) LOG.debug( "Wait on namespace=(%s) labels=(%s) for %s sec " "(k8s wait %s times, sleep %ss)", namespace, label_selector, timeout, wait_attempts, sleep_time) if not namespace: # This shouldn't be reachable LOG.warn('"namespace" not specified, waiting across all available ' 'namespaces is likely to cause unintended consequences.') if not label_selector: LOG.warn('"label_selector" not specified, waiting with no labels ' 'may cause unintended consequences.') deadline = time.time() + timeout # NOTE(mark-burnett): Attempt to wait multiple times without # modification, in case new pods appear after our watch exits. successes = 0 while successes < wait_attempts: deadline_remaining = int(round(deadline - time.time())) if deadline_remaining <= 0: return False timed_out, modified_pods, unready_pods = self._wait_one_time( namespace=namespace, label_selector=label_selector, timeout=deadline_remaining) if timed_out: LOG.info('Timed out waiting for pods: %s', sorted(unready_pods)) raise exceptions.KubernetesWatchTimeoutException( 'Timed out while waiting on namespace=(%s) labels=(%s)' % (namespace, label_selector)) return False if modified_pods: successes = 0 LOG.debug('Continuing to wait, found modified pods: %s', sorted(modified_pods)) else: successes += 1 LOG.debug('Found no modified pods this attempt. successes=%d', successes) time.sleep(sleep_time) return True
def wait(self, timeout): ''' :param timeout: time before disconnecting ``Watch`` stream ''' LOG.info( "Waiting for resource type=%s, namespace=%s labels=%s for %ss " "(k8s wait %s times, sleep %ss)", self.resource_type, self.chart_wait.namespace, self.label_selector, timeout, self.chart_wait.k8s_wait_attempts, self.chart_wait.k8s_wait_attempt_sleep) if not self.label_selector: LOG.warn('"label_selector" not specified, waiting with no labels ' 'may cause unintended consequences.') # Track the overall deadline for timing out during waits deadline = time.time() + timeout # NOTE(mark-burnett): Attempt to wait multiple times without # modification, in case new resources appear after our watch exits. successes = 0 while True: deadline_remaining = int(round(deadline - time.time())) if deadline_remaining <= 0: error = ( "Timed out waiting for resource type={}, namespace={}, " "labels={}".format(self.resource_type, self.chart_wait.namespace, self.label_selector)) LOG.error(error) raise k8s_exceptions.KubernetesWatchTimeoutException(error) timed_out, modified, unready, found_resources = ( self._watch_resource_completions(timeout=deadline_remaining)) if not found_resources: if self.skip_if_none_found: return else: LOG.warn( 'Saw no resources for ' 'resource type=%s, namespace=%s, labels=(%s). Are the ' 'labels correct?', self.resource_type, self.chart_wait.namespace, self.label_selector) # TODO(seaneagan): Should probably fail here even when resources # were not found, at least once we have an option to ignore # wait timeouts. if timed_out and found_resources: error = "Timed out waiting for resources={}".format( sorted(unready)) LOG.error(error) raise k8s_exceptions.KubernetesWatchTimeoutException(error) if modified: successes = 0 LOG.debug('Found modified resources: %s', sorted(modified)) else: successes += 1 LOG.debug('Found no modified resources.') if successes >= self.chart_wait.k8s_wait_attempts: break LOG.debug( 'Continuing to wait: %s consecutive attempts without ' 'modified resources of %s required.', successes, self.chart_wait.k8s_wait_attempts) time.sleep(self.chart_wait.k8s_wait_attempt_sleep) return True
def wait(self, timeout): ''' :param timeout: time before disconnecting ``Watch`` stream ''' LOG.info( "Waiting for resource type=%s, namespace=%s labels=%s for %ss " "(k8s wait %s times, sleep %ss)", self.resource_type, self.chart_wait.namespace, self.label_selector, timeout, self.chart_wait.k8s_wait_attempts, self.chart_wait.k8s_wait_attempt_sleep) if not self.label_selector: LOG.warn('"label_selector" not specified, waiting with no labels ' 'may cause unintended consequences.') # Track the overall deadline for timing out during waits deadline = time.time() + timeout # NOTE(mark-burnett): Attempt to wait multiple times without # modification, in case new resources appear after our watch exits. successes = 0 while True: deadline_remaining = int(round(deadline - time.time())) if deadline_remaining <= 0: error = ( "Timed out waiting for resource type={}, namespace={}, " "labels={}".format(self.resource_type, self.chart_wait.namespace, self.label_selector)) LOG.error(error) raise k8s_exceptions.KubernetesWatchTimeoutException(error) timed_out, modified, unready, found_resources = ( self._watch_resource_completions(timeout=deadline_remaining)) if (not found_resources) and self.skip_if_none_found: return if timed_out: if not found_resources: details = ( 'None found! Are `wait.labels` correct? Does ' '`wait.resources` need to exclude `type: {}`?'.format( self.resource_type)) else: details = ('These {}s were not ready={}'.format( self.resource_type, sorted(unready))) error = ( 'Timed out waiting for {}s (namespace={}, labels=({})). {}' .format(self.resource_type, self.chart_wait.namespace, self.label_selector, details)) LOG.error(error) raise k8s_exceptions.KubernetesWatchTimeoutException(error) if modified: successes = 0 LOG.debug('Found modified resources: %s', sorted(modified)) else: successes += 1 LOG.debug('Found no modified resources.') if successes >= self.chart_wait.k8s_wait_attempts: break LOG.debug( 'Continuing to wait: %s consecutive attempts without ' 'modified resources of %s required.', successes, self.chart_wait.k8s_wait_attempts) time.sleep(self.chart_wait.k8s_wait_attempt_sleep) return True