def delete_test_pods(self): """Deletes any existing test pods for the release, as identified by the wait labels for the chart, to avoid test pod name conflicts when creating the new test pod as well as just for general cleanup since the new test pod should supercede it. """ labels = get_wait_labels(self.chart) # Guard against labels being left empty, so we don't delete other # chart's test pods. if labels: label_selector = label_selectors(labels) namespace = self.chart['namespace'] list_args = { 'namespace': namespace, 'label_selector': label_selector, 'timeout_seconds': self.k8s_timeout } pod_list = self.tiller.k8s.client.list_namespaced_pod(**list_args) test_pods = [pod for pod in pod_list.items if is_test_pod(pod)] if test_pods: LOG.info( 'Found existing test pods for release with ' 'namespace=%s, labels=(%s)', namespace, label_selector) for test_pod in test_pods: pod_name = test_pod.metadata.name LOG.info('Deleting existing test pod: %s', pod_name) self.tiller.k8s.delete_pod_action(pod_name, namespace, timeout=self.k8s_timeout)
def __init__( self, resource_type, chart_wait, labels, get_resources, required=True): self.resource_type = resource_type self.chart_wait = chart_wait self.label_selector = label_selectors(labels) self.get_resources = get_resources self.required = required
def rolling_upgrade_pod_deployment( self, name, release_name, namespace, resource_labels, action_type, chart, disable_hooks, values, timeout=const.DEFAULT_TILLER_TIMEOUT): ''' update statefulsets (daemon, stateful) ''' if action_type == 'daemonset': LOG.info('Updating: %s', action_type) label_selector = '' if resource_labels is not None: label_selector = label_selectors(resource_labels) get_daemonset = self.k8s.get_namespace_daemon_set( namespace, label_selector=label_selector) for ds in get_daemonset.items: ds_name = ds.metadata.name ds_labels = ds.metadata.labels if ds_name == name: LOG.info( "Deleting %s : %s in %s", action_type, ds_name, namespace) self.k8s.delete_daemon_action(ds_name, namespace) # update the daemonset yaml template = self.get_chart_templates( ds_name, name, release_name, namespace, chart, disable_hooks, values) template['metadata']['labels'] = ds_labels template['spec']['template']['metadata'][ 'labels'] = ds_labels self.k8s.create_daemon_action( namespace=namespace, template=template) # delete pods self.delete_resources( 'pod', resource_labels, namespace, wait=True, timeout=timeout) else: LOG.error("Unable to exectue name: % type: %s", name, action_type)
def __init__(self, resource_type, chart_wait, labels, get_resources, skip_if_none_found=False): self.resource_type = resource_type self.chart_wait = chart_wait self.label_selector = label_selectors(labels) self.get_resources = get_resources self.skip_if_none_found = skip_if_none_found
def wait_until_ready(self, release=None, namespace='default', labels='', timeout=300, sleep=15): ''' :param release - part of namespace :param timeout - time before disconnecting stream ''' LOG.debug("Wait on %s for %s sec", namespace, timeout) label_selector = '' if labels: label_selector = label_selectors(labels) valid_state = ['Succeeded', 'Running'] wait_timeout = time.time() + 60 * timeout while True: self.is_pods_ready(label_selector=label_selector, timeout=timeout) pod_ready = [] not_ready = [] for pod in self.client.list_pod_for_all_namespaces( label_selector=label_selector).items: p_state = pod.status.phase p_name = pod.metadata.name if p_state in valid_state: pod_ready.append(True) continue pod_ready.append(False) not_ready.append(p_name) LOG.debug('%s', p_state) if time.time() > wait_timeout or all(pod_ready): LOG.debug("Pod States %s", pod_ready) break if time.time() > wait_timeout and not all(pod_ready): LOG.exception('Failed to bring up release %s: %s', release, not_ready) break else: LOG.debug('time: %s pod %s', wait_timeout, pod_ready)
def wait_until_ready(self, release=None, namespace='default', labels='', timeout=300, sleep=15, required_successes=3, inter_success_wait=10): ''' :param release - part of namespace :param timeout - time before disconnecting stream ''' label_selector = '' if labels: label_selector = label_selectors(labels) LOG.debug("Wait on %s (%s) for %s sec", namespace, label_selector, timeout) deadline = time.time() + timeout # NOTE(mark-burnett): Attempt to wait multiple times without # modification, in case new pods appear after our watch exits. successes = 0 while successes < required_successes: deadline_remaining = int(deadline - time.time()) if deadline_remaining <= 0: return False timed_out, modified_pods, unready_pods = self.wait_one_time( label_selector, timeout=deadline_remaining) if timed_out: LOG.info('Timed out waiting for pods: %s', unready_pods) return False if modified_pods: successes = 0 LOG.debug('Continuing to wait, found modified pods: %s', modified_pods) else: successes += 1 LOG.debug('Found no modified pods this attempt. successes=%d', successes) time.sleep(inter_success_wait) return True
def delete_resources(self, release_name, resource_name, resource_type, resource_labels, namespace, wait=False): ''' :params release_name - release name the specified resource is under :params resource_name - name of specific resource :params resource_type - type of resource e.g. job, pod, etc. :params resource_labels - labels by which to identify the resource :params namespace - namespace of the resource Apply deletion logic based on type of resource ''' label_selector = '' if resource_labels is not None: label_selector = label_selectors(resource_labels) LOG.debug( "Deleting resources in namespace %s matching" "selectors %s.", namespace, label_selector) if 'job' in resource_type: get_jobs = self.k8s.get_namespace_job(namespace, label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name LOG.info("Deleting %s in namespace: %s", jb_name, namespace) self.k8s.delete_job_action(jb_name, namespace) elif 'pod' in resource_type: release_pods = self.k8s.get_namespace_pod(namespace, label_selector) for pod in release_pods.items: pod_name = pod.metadata.name LOG.info("Deleting %s in namespace: %s", pod_name, namespace) self.k8s.delete_namespace_pod(pod_name, namespace) if wait: self.k8s.wait_for_pod_redeployment(pod_name, namespace) else: LOG.error("Unable to execute name: %s type: %s ", resource_name, resource_type)
def wait_until_ready(self, release=None, namespace='', labels='', timeout=DEFAULT_K8S_TIMEOUT, k8s_wait_attempts=1, k8s_wait_attempt_sleep=1): ''' Wait until all pods become ready given the filters provided by ``release``, ``labels`` and ``namespace``. :param release: chart release :param namespace: the namespace used to filter which pods to wait on :param labels: the labels used to filter which pods to wait on :param timeout: time before disconnecting ``Watch`` stream :param k8s_wait_attempts: The number of times to attempt waiting for pods to become ready (minimum 1). :param k8s_wait_attempt_sleep: The time in seconds to sleep between attempts (minimum 1). ''' timeout = self._check_timeout(timeout) # NOTE(MarshM) 'release' is currently unused label_selector = label_selectors(labels) if labels else '' wait_attempts = (k8s_wait_attempts if k8s_wait_attempts >= 1 else 1) sleep_time = (k8s_wait_attempt_sleep if k8s_wait_attempt_sleep >= 1 else 1) LOG.debug( "Wait on namespace=(%s) labels=(%s) for %s sec " "(k8s wait %s times, sleep %ss)", namespace, label_selector, timeout, wait_attempts, sleep_time) if not namespace: # This shouldn't be reachable LOG.warn('"namespace" not specified, waiting across all available ' 'namespaces is likely to cause unintended consequences.') if not label_selector: LOG.warn('"label_selector" not specified, waiting with no labels ' 'may cause unintended consequences.') # Track the overall deadline for timing out during waits deadline = time.time() + timeout # First, we should watch for jobs before checking pods, as a job can # still be running even after its current pods look healthy or have # been removed and are pending reschedule found_jobs = self.get_namespace_job(namespace, label_selector) if len(found_jobs.items): self._watch_job_completion(namespace, label_selector, timeout) # NOTE(mark-burnett): Attempt to wait multiple times without # modification, in case new pods appear after our watch exits. successes = 0 while successes < wait_attempts: deadline_remaining = int(round(deadline - time.time())) if deadline_remaining <= 0: LOG.info('Timed out while waiting for pods.') raise exceptions.KubernetesWatchTimeoutException( 'Timed out while waiting on namespace=(%s) labels=(%s)' % (namespace, label_selector)) timed_out, modified_pods, unready_pods, found_events = ( self._watch_pod_completions(namespace=namespace, label_selector=label_selector, timeout=deadline_remaining)) if not found_events: LOG.warn( 'Saw no install/update events for release=%s, ' 'namespace=%s, labels=(%s). Are the labels correct?', release, namespace, label_selector) if timed_out: LOG.info('Timed out waiting for pods: %s', sorted(unready_pods)) raise exceptions.KubernetesWatchTimeoutException( 'Timed out while waiting on namespace=(%s) labels=(%s)' % (namespace, label_selector)) if modified_pods: successes = 0 LOG.debug('Continuing to wait, found modified pods: %s', sorted(modified_pods)) else: successes += 1 LOG.debug('Found no modified pods this attempt. successes=%d', successes) time.sleep(sleep_time) return True
def delete_resources(self, release_name, resource_name, resource_type, resource_labels, namespace, wait=False, timeout=const.DEFAULT_TILLER_TIMEOUT): ''' :param release_name: release name the specified resource is under :param resource_name: name of specific resource :param resource_type: type of resource e.g. job, pod, etc. :param resource_labels: labels by which to identify the resource :param namespace: namespace of the resource Apply deletion logic based on type of resource ''' timeout = self._check_timeout(wait, timeout) label_selector = '' if resource_labels is not None: label_selector = label_selectors(resource_labels) LOG.debug( "Deleting resources in namespace %s matching " "selectors (%s).", namespace, label_selector) handled = False if resource_type == 'job': get_jobs = self.k8s.get_namespace_job( namespace, label_selector=label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name if self.dry_run: LOG.info( 'Skipping delete job during `dry-run`, would ' 'have deleted job %s in namespace=%s.', jb_name, namespace) continue LOG.info("Deleting job %s in namespace: %s", jb_name, namespace) self.k8s.delete_job_action(jb_name, namespace, timeout=timeout) handled = True if resource_type == 'cronjob' or resource_type == 'job': get_jobs = self.k8s.get_namespace_cron_job( namespace, label_selector=label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name if resource_type == 'job': # TODO: Eventually disallow this, allowing initially since # some existing clients were expecting this behavior. LOG.warn("Deleting cronjobs via `type: job` is " "deprecated, use `type: cronjob` instead") if self.dry_run: LOG.info( 'Skipping delete cronjob during `dry-run`, would ' 'have deleted cronjob %s in namespace=%s.', jb_name, namespace) continue LOG.info("Deleting cronjob %s in namespace: %s", jb_name, namespace) self.k8s.delete_cron_job_action(jb_name, namespace) handled = True if resource_type == 'pod': release_pods = self.k8s.get_namespace_pod( namespace, label_selector=label_selector) for pod in release_pods.items: pod_name = pod.metadata.name if self.dry_run: LOG.info( 'Skipping delete pod during `dry-run`, would ' 'have deleted pod %s in namespace=%s.', pod_name, namespace) continue LOG.info("Deleting pod %s in namespace: %s", pod_name, namespace) self.k8s.delete_pod_action(pod_name, namespace) if wait: self.k8s.wait_for_pod_redeployment(pod_name, namespace) handled = True if not handled: LOG.error("Unable to execute name: %s type: %s ", resource_name, resource_type)
def wait_until_ready(self, release=None, namespace='', labels='', timeout=DEFAULT_K8S_TIMEOUT, k8s_wait_attempts=1, k8s_wait_attempt_sleep=1): ''' Wait until all pods become ready given the filters provided by ``release``, ``labels`` and ``namespace``. :param release: chart release :param namespace: the namespace used to filter which pods to wait on :param labels: the labels used to filter which pods to wait on :param timeout: time before disconnecting ``Watch`` stream :param k8s_wait_attempts: The number of times to attempt waiting for pods to become ready (minimum 1). :param k8s_wait_attempt_sleep: The time in seconds to sleep between attempts (minimum 1). ''' # NOTE(MarshM) 'release' is currently unused label_selector = label_selectors(labels) if labels else '' wait_attempts = (k8s_wait_attempts if k8s_wait_attempts >= 1 else 1) sleep_time = (k8s_wait_attempt_sleep if k8s_wait_attempt_sleep >= 1 else 1) LOG.debug( "Wait on namespace=(%s) labels=(%s) for %s sec " "(k8s wait %s times, sleep %ss)", namespace, label_selector, timeout, wait_attempts, sleep_time) if not namespace: # This shouldn't be reachable LOG.warn('"namespace" not specified, waiting across all available ' 'namespaces is likely to cause unintended consequences.') if not label_selector: LOG.warn('"label_selector" not specified, waiting with no labels ' 'may cause unintended consequences.') deadline = time.time() + timeout # NOTE(mark-burnett): Attempt to wait multiple times without # modification, in case new pods appear after our watch exits. successes = 0 while successes < wait_attempts: deadline_remaining = int(round(deadline - time.time())) if deadline_remaining <= 0: return False timed_out, modified_pods, unready_pods = self._wait_one_time( namespace=namespace, label_selector=label_selector, timeout=deadline_remaining) if timed_out: LOG.info('Timed out waiting for pods: %s', sorted(unready_pods)) raise exceptions.KubernetesWatchTimeoutException( 'Timed out while waiting on namespace=(%s) labels=(%s)' % (namespace, label_selector)) return False if modified_pods: successes = 0 LOG.debug('Continuing to wait, found modified pods: %s', sorted(modified_pods)) else: successes += 1 LOG.debug('Found no modified pods this attempt. successes=%d', successes) time.sleep(sleep_time) return True
def delete_resources(self, resource_type, resource_labels, namespace, wait=False, timeout=const.DEFAULT_TILLER_TIMEOUT): ''' Delete resources matching provided resource type, labels, and namespace. :param resource_type: type of resource e.g. job, pod, etc. :param resource_labels: labels for selecting the resources :param namespace: namespace of resources ''' timeout = self._check_timeout(wait, timeout) label_selector = '' if resource_labels is not None: label_selector = label_selectors(resource_labels) LOG.debug( "Deleting resources in namespace %s matching " "selectors (%s).", namespace, label_selector) handled = False if resource_type == 'job': get_jobs = self.k8s.get_namespace_job( namespace, label_selector=label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name if self.dry_run: LOG.info( 'Skipping delete job during `dry-run`, would ' 'have deleted job %s in namespace=%s.', jb_name, namespace) continue LOG.info("Deleting job %s in namespace: %s", jb_name, namespace) self.k8s.delete_job_action(jb_name, namespace, timeout=timeout) handled = True # TODO: Remove when v1 doc support is removed. chart = get_current_chart() schema_info = schema.get_schema_info(chart['schema']) job_implies_cronjob = schema_info.version < 2 implied_cronjob = resource_type == 'job' and job_implies_cronjob if resource_type == 'cronjob' or implied_cronjob: get_jobs = self.k8s.get_namespace_cron_job( namespace, label_selector=label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name # TODO: Remove when v1 doc support is removed. if implied_cronjob: LOG.warn("Deleting cronjobs via `type: job` is " "deprecated, use `type: cronjob` instead") if self.dry_run: LOG.info( 'Skipping delete cronjob during `dry-run`, would ' 'have deleted cronjob %s in namespace=%s.', jb_name, namespace) continue LOG.info("Deleting cronjob %s in namespace: %s", jb_name, namespace) self.k8s.delete_cron_job_action(jb_name, namespace) handled = True if resource_type == 'pod': release_pods = self.k8s.get_namespace_pod( namespace, label_selector=label_selector) for pod in release_pods.items: pod_name = pod.metadata.name if self.dry_run: LOG.info( 'Skipping delete pod during `dry-run`, would ' 'have deleted pod %s in namespace=%s.', pod_name, namespace) continue LOG.info("Deleting pod %s in namespace: %s", pod_name, namespace) self.k8s.delete_pod_action(pod_name, namespace) if wait: self.k8s.wait_for_pod_redeployment(pod_name, namespace) handled = True if not handled: LOG.error('No resources found with labels=%s type=%s namespace=%s', resource_labels, resource_type, namespace)
def delete_resources(self, release_name, resource_name, resource_type, resource_labels, namespace, wait=False, timeout=TILLER_TIMEOUT): ''' :params release_name - release name the specified resource is under :params resource_name - name of specific resource :params resource_type - type of resource e.g. job, pod, etc. :params resource_labels - labels by which to identify the resource :params namespace - namespace of the resource Apply deletion logic based on type of resource ''' label_selector = '' if resource_labels is not None: label_selector = label_selectors(resource_labels) LOG.debug( "Deleting resources in namespace %s matching " "selectors %s.", namespace, label_selector) handled = False if resource_type == 'job': get_jobs = self.k8s.get_namespace_job(namespace, label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name LOG.info("Deleting job %s in namespace: %s", jb_name, namespace) self.k8s.delete_job_action(jb_name, namespace, timeout=timeout) handled = True if resource_type == 'cronjob' or resource_type == 'job': get_jobs = self.k8s.get_namespace_cron_job(namespace, label_selector) for jb in get_jobs.items: jb_name = jb.metadata.name LOG.info("Deleting cron job %s in namespace: %s", jb_name, namespace) if resource_type == 'job': # TODO: Eventually disallow this, allowing initially since # some existing clients were expecting this behavior. LOG.warning("Deleting cron jobs via `type: job` is " "deprecated, use `type: cronjob` instead") self.k8s.delete_cron_job_action(jb_name, namespace) handled = True if resource_type == 'pod': release_pods = self.k8s.get_namespace_pod(namespace, label_selector) for pod in release_pods.items: pod_name = pod.metadata.name LOG.info("Deleting pod %s in namespace: %s", pod_name, namespace) self.k8s.delete_namespace_pod(pod_name, namespace) if wait: self.k8s.wait_for_pod_redeployment(pod_name, namespace) handled = True if not handled: LOG.error("Unable to execute name: %s type: %s ", resource_name, resource_type)