def _wait(self, deadline):
'''
Waits for resources to become ready.
Returns whether resources were modified, or `None` if that is to be
ignored.
'''
deadline_remaining = int(round(deadline - time.time()))
if deadline_remaining <= 0:
error = (
"Timed out waiting for resource type={}, namespace={}, "
"labels={}".format(
self.resource_type, self.chart_wait.release_id.namespace,
self.label_selector))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
timed_out, modified, unready, found_resources = (
self._watch_resource_completions(timeout=deadline_remaining))
if (not found_resources) and not self.required:
return None
if timed_out:
if not found_resources:
details = (
'None found! Are `wait.labels` correct? Does '
'`wait.resources` need to exclude `type: {}`?'.format(
self.resource_type))
else:
details = (
'These {}s were not ready={}'.format(
self.resource_type, sorted(unready)))
error = (
'Timed out waiting for {}s (namespace={}, labels=({})). {}'.
format(
self.resource_type, self.chart_wait.release_id.namespace,
self.label_selector, details))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
return modified
def _watch_job_completion(self, namespace, label_selector, timeout):
'''
Watch and wait for job completion.
Returns when conditions are met, or raises a timeout exception.
'''
try:
timeout = self._check_timeout(timeout)
ready_jobs = {}
w = watch.Watch()
for event in w.stream(self.batch_api.list_namespaced_job,
namespace=namespace,
label_selector=label_selector,
timeout_seconds=timeout):
job_name = event['object'].metadata.name
LOG.debug('Watch event %s on job %s', event['type'].upper(),
job_name)
# Track the expected and actual number of completed pods
# See: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/ # noqa
expected = event['object'].spec.completions
completed = event['object'].status.succeeded
if expected != completed:
ready_jobs[job_name] = False
else:
ready_jobs[job_name] = True
LOG.debug(
'Job %s complete (spec.completions=%s, '
'status.succeeded=%s)', job_name, expected, completed)
if all(ready_jobs.values()):
return True
except ApiException as e:
LOG.exception(
"Exception when watching jobs: namespace=%s, labels=(%s)",
namespace, label_selector)
raise e
if not ready_jobs:
LOG.warn(
'Saw no job events for namespace=%s, labels=(%s). '
'Are the labels correct?', namespace, label_selector)
return False
err_msg = ('Reached timeout while waiting for job completions: '
'namespace=%s, labels=(%s)' % (namespace, label_selector))
LOG.error(err_msg)
raise exceptions.KubernetesWatchTimeoutException(err_msg)
def _delete_job_action(self,
list_func,
delete_func,
job_type_description,
name,
namespace="default",
propagation_policy='Foreground',
timeout=DEFAULT_K8S_TIMEOUT):
try:
LOG.debug('Deleting %s %s, Wait timeout=%s', job_type_description,
name, timeout)
body = client.V1DeleteOptions()
w = watch.Watch()
issue_delete = True
for event in w.stream(list_func,
namespace=namespace,
timeout_seconds=timeout):
if issue_delete:
delete_func(name=name,
namespace=namespace,
body=body,
propagation_policy=propagation_policy)
issue_delete = False
event_type = event['type'].upper()
job_name = event['object'].metadata.name
if event_type == 'DELETED' and job_name == name:
LOG.debug('Successfully deleted %s %s',
job_type_description, job_name)
return
err_msg = ('Reached timeout while waiting to delete %s: '
'name=%s, namespace=%s' %
(job_type_description, name, namespace))
LOG.error(err_msg)
raise exceptions.KubernetesWatchTimeoutException(err_msg)
except ApiException as e:
LOG.exception("Exception when deleting %s: name=%s, namespace=%s",
job_type_description, name, namespace)
raise e
def _delete_item_action(self,
list_func,
delete_func,
object_type_description,
name,
namespace="default",
propagation_policy='Foreground',
timeout=DEFAULT_K8S_TIMEOUT):
'''
This function takes the action to delete an object (job, cronjob, pod)
from kubernetes. It will wait for the object to be fully deleted before
returning to processing or timing out.
:param list_func: The callback function to list the specified object
type
:param delete_func: The callback function to delete the specified
object type
:param object_type_description: The types of objects to delete,
in `job`, `cronjob`, or `pod`
:param name: The name of the object to delete
:param namespace: The namespace of the object
:param propagation_policy: The Kubernetes propagation_policy to apply
to the delete. Default 'Foreground' means that child objects
will be deleted before the given object is marked as deleted.
See: https://kubernetes.io/docs/concepts/workloads/controllers/garbage-collection/#controlling-how-the-garbage-collector-deletes-dependents # noqa
:param timeout: The timeout to wait for the delete to complete
'''
try:
timeout = self._check_timeout(timeout)
LOG.debug('Watching to delete %s %s, Wait timeout=%s',
object_type_description, name, timeout)
body = client.V1DeleteOptions()
w = watch.Watch()
issue_delete = True
found_events = False
for event in w.stream(list_func,
namespace=namespace,
timeout_seconds=timeout):
if issue_delete:
delete_func(name=name,
namespace=namespace,
body=body,
propagation_policy=propagation_policy)
issue_delete = False
event_type = event['type'].upper()
item_name = event['object'].metadata.name
LOG.debug('Watch event %s on %s', event_type, item_name)
if item_name == name:
found_events = True
if event_type == 'DELETED':
LOG.info('Successfully deleted %s %s',
object_type_description, item_name)
return
if not found_events:
LOG.warn('Saw no delete events for %s %s in namespace=%s',
object_type_description, name, namespace)
err_msg = ('Reached timeout while waiting to delete %s: '
'name=%s, namespace=%s' %
(object_type_description, name, namespace))
LOG.error(err_msg)
raise exceptions.KubernetesWatchTimeoutException(err_msg)
except ApiException as e:
LOG.exception("Exception when deleting %s: name=%s, namespace=%s",
object_type_description, name, namespace)
raise e
def wait_until_ready(self,
release=None,
namespace='',
labels='',
timeout=DEFAULT_K8S_TIMEOUT,
k8s_wait_attempts=1,
k8s_wait_attempt_sleep=1):
'''
Wait until all pods become ready given the filters provided by
``release``, ``labels`` and ``namespace``.
:param release: chart release
:param namespace: the namespace used to filter which pods to wait on
:param labels: the labels used to filter which pods to wait on
:param timeout: time before disconnecting ``Watch`` stream
:param k8s_wait_attempts: The number of times to attempt waiting
for pods to become ready (minimum 1).
:param k8s_wait_attempt_sleep: The time in seconds to sleep
between attempts (minimum 1).
'''
timeout = self._check_timeout(timeout)
# NOTE(MarshM) 'release' is currently unused
label_selector = label_selectors(labels) if labels else ''
wait_attempts = (k8s_wait_attempts if k8s_wait_attempts >= 1 else 1)
sleep_time = (k8s_wait_attempt_sleep
if k8s_wait_attempt_sleep >= 1 else 1)
LOG.debug(
"Wait on namespace=(%s) labels=(%s) for %s sec "
"(k8s wait %s times, sleep %ss)", namespace, label_selector,
timeout, wait_attempts, sleep_time)
if not namespace:
# This shouldn't be reachable
LOG.warn('"namespace" not specified, waiting across all available '
'namespaces is likely to cause unintended consequences.')
if not label_selector:
LOG.warn('"label_selector" not specified, waiting with no labels '
'may cause unintended consequences.')
# Track the overall deadline for timing out during waits
deadline = time.time() + timeout
# First, we should watch for jobs before checking pods, as a job can
# still be running even after its current pods look healthy or have
# been removed and are pending reschedule
found_jobs = self.get_namespace_job(namespace, label_selector)
if len(found_jobs.items):
self._watch_job_completion(namespace, label_selector, timeout)
# NOTE(mark-burnett): Attempt to wait multiple times without
# modification, in case new pods appear after our watch exits.
successes = 0
while successes < wait_attempts:
deadline_remaining = int(round(deadline - time.time()))
if deadline_remaining <= 0:
LOG.info('Timed out while waiting for pods.')
raise exceptions.KubernetesWatchTimeoutException(
'Timed out while waiting on namespace=(%s) labels=(%s)' %
(namespace, label_selector))
timed_out, modified_pods, unready_pods, found_events = (
self._watch_pod_completions(namespace=namespace,
label_selector=label_selector,
timeout=deadline_remaining))
if not found_events:
LOG.warn(
'Saw no install/update events for release=%s, '
'namespace=%s, labels=(%s). Are the labels correct?',
release, namespace, label_selector)
if timed_out:
LOG.info('Timed out waiting for pods: %s',
sorted(unready_pods))
raise exceptions.KubernetesWatchTimeoutException(
'Timed out while waiting on namespace=(%s) labels=(%s)' %
(namespace, label_selector))
if modified_pods:
successes = 0
LOG.debug('Continuing to wait, found modified pods: %s',
sorted(modified_pods))
else:
successes += 1
LOG.debug('Found no modified pods this attempt. successes=%d',
successes)
time.sleep(sleep_time)
return True
def wait_until_ready(self,
release=None,
namespace='',
labels='',
timeout=DEFAULT_K8S_TIMEOUT,
k8s_wait_attempts=1,
k8s_wait_attempt_sleep=1):
'''
Wait until all pods become ready given the filters provided by
``release``, ``labels`` and ``namespace``.
:param release: chart release
:param namespace: the namespace used to filter which pods to wait on
:param labels: the labels used to filter which pods to wait on
:param timeout: time before disconnecting ``Watch`` stream
:param k8s_wait_attempts: The number of times to attempt waiting
for pods to become ready (minimum 1).
:param k8s_wait_attempt_sleep: The time in seconds to sleep
between attempts (minimum 1).
'''
# NOTE(MarshM) 'release' is currently unused
label_selector = label_selectors(labels) if labels else ''
wait_attempts = (k8s_wait_attempts if k8s_wait_attempts >= 1 else 1)
sleep_time = (k8s_wait_attempt_sleep
if k8s_wait_attempt_sleep >= 1 else 1)
LOG.debug(
"Wait on namespace=(%s) labels=(%s) for %s sec "
"(k8s wait %s times, sleep %ss)", namespace, label_selector,
timeout, wait_attempts, sleep_time)
if not namespace:
# This shouldn't be reachable
LOG.warn('"namespace" not specified, waiting across all available '
'namespaces is likely to cause unintended consequences.')
if not label_selector:
LOG.warn('"label_selector" not specified, waiting with no labels '
'may cause unintended consequences.')
deadline = time.time() + timeout
# NOTE(mark-burnett): Attempt to wait multiple times without
# modification, in case new pods appear after our watch exits.
successes = 0
while successes < wait_attempts:
deadline_remaining = int(round(deadline - time.time()))
if deadline_remaining <= 0:
return False
timed_out, modified_pods, unready_pods = self._wait_one_time(
namespace=namespace,
label_selector=label_selector,
timeout=deadline_remaining)
if timed_out:
LOG.info('Timed out waiting for pods: %s',
sorted(unready_pods))
raise exceptions.KubernetesWatchTimeoutException(
'Timed out while waiting on namespace=(%s) labels=(%s)' %
(namespace, label_selector))
return False
if modified_pods:
successes = 0
LOG.debug('Continuing to wait, found modified pods: %s',
sorted(modified_pods))
else:
successes += 1
LOG.debug('Found no modified pods this attempt. successes=%d',
successes)
time.sleep(sleep_time)
return True
def wait(self, timeout):
'''
:param timeout: time before disconnecting ``Watch`` stream
'''
LOG.info(
"Waiting for resource type=%s, namespace=%s labels=%s for %ss "
"(k8s wait %s times, sleep %ss)", self.resource_type,
self.chart_wait.namespace, self.label_selector, timeout,
self.chart_wait.k8s_wait_attempts,
self.chart_wait.k8s_wait_attempt_sleep)
if not self.label_selector:
LOG.warn('"label_selector" not specified, waiting with no labels '
'may cause unintended consequences.')
# Track the overall deadline for timing out during waits
deadline = time.time() + timeout
# NOTE(mark-burnett): Attempt to wait multiple times without
# modification, in case new resources appear after our watch exits.
successes = 0
while True:
deadline_remaining = int(round(deadline - time.time()))
if deadline_remaining <= 0:
error = (
"Timed out waiting for resource type={}, namespace={}, "
"labels={}".format(self.resource_type,
self.chart_wait.namespace,
self.label_selector))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
timed_out, modified, unready, found_resources = (
self._watch_resource_completions(timeout=deadline_remaining))
if not found_resources:
if self.skip_if_none_found:
return
else:
LOG.warn(
'Saw no resources for '
'resource type=%s, namespace=%s, labels=(%s). Are the '
'labels correct?', self.resource_type,
self.chart_wait.namespace, self.label_selector)
# TODO(seaneagan): Should probably fail here even when resources
# were not found, at least once we have an option to ignore
# wait timeouts.
if timed_out and found_resources:
error = "Timed out waiting for resources={}".format(
sorted(unready))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
if modified:
successes = 0
LOG.debug('Found modified resources: %s', sorted(modified))
else:
successes += 1
LOG.debug('Found no modified resources.')
if successes >= self.chart_wait.k8s_wait_attempts:
break
LOG.debug(
'Continuing to wait: %s consecutive attempts without '
'modified resources of %s required.', successes,
self.chart_wait.k8s_wait_attempts)
time.sleep(self.chart_wait.k8s_wait_attempt_sleep)
return True
def wait(self, timeout):
'''
:param timeout: time before disconnecting ``Watch`` stream
'''
LOG.info(
"Waiting for resource type=%s, namespace=%s labels=%s for %ss "
"(k8s wait %s times, sleep %ss)", self.resource_type,
self.chart_wait.namespace, self.label_selector, timeout,
self.chart_wait.k8s_wait_attempts,
self.chart_wait.k8s_wait_attempt_sleep)
if not self.label_selector:
LOG.warn('"label_selector" not specified, waiting with no labels '
'may cause unintended consequences.')
# Track the overall deadline for timing out during waits
deadline = time.time() + timeout
# NOTE(mark-burnett): Attempt to wait multiple times without
# modification, in case new resources appear after our watch exits.
successes = 0
while True:
deadline_remaining = int(round(deadline - time.time()))
if deadline_remaining <= 0:
error = (
"Timed out waiting for resource type={}, namespace={}, "
"labels={}".format(self.resource_type,
self.chart_wait.namespace,
self.label_selector))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
timed_out, modified, unready, found_resources = (
self._watch_resource_completions(timeout=deadline_remaining))
if (not found_resources) and self.skip_if_none_found:
return
if timed_out:
if not found_resources:
details = (
'None found! Are `wait.labels` correct? Does '
'`wait.resources` need to exclude `type: {}`?'.format(
self.resource_type))
else:
details = ('These {}s were not ready={}'.format(
self.resource_type, sorted(unready)))
error = (
'Timed out waiting for {}s (namespace={}, labels=({})). {}'
.format(self.resource_type, self.chart_wait.namespace,
self.label_selector, details))
LOG.error(error)
raise k8s_exceptions.KubernetesWatchTimeoutException(error)
if modified:
successes = 0
LOG.debug('Found modified resources: %s', sorted(modified))
else:
successes += 1
LOG.debug('Found no modified resources.')
if successes >= self.chart_wait.k8s_wait_attempts:
break
LOG.debug(
'Continuing to wait: %s consecutive attempts without '
'modified resources of %s required.', successes,
self.chart_wait.k8s_wait_attempts)
time.sleep(self.chart_wait.k8s_wait_attempt_sleep)
return True
