def check_logs(k8s_client): pod_status = 'Running' all_pods = k8s_client.list_pod_for_all_namespaces() for pod in all_pods.items: pod_name = pod.metadata.name namespace = pod.metadata.namespace for container in pod.spec.containers: if container.name in SKIP_CONTAINERS: continue if pod.status.phase != pod_status: # Wait for the Pod to be ready utils.retry( kube_utils.check_pod_status( k8s_client, name=pod_name, namespace=namespace, state=pod_status ), times=12, wait=5, name="wait for Pod '{}'".format(pod_name), ) logs = k8s_client.read_namespaced_pod_log( pod_name, namespace, container=container.name ) assert logs.strip(), ( "Couldn't find logs for container '{}' in Pod '{}' (status {})" ).format(container.name, pod_name, pod.status.phase)
def check_file_content_inside_pod(volume_name, path, content, k8s_client): name = '{}-pod'.format(volume_name) def _check_file_content(): try: result = k8s.stream.stream( k8s_client.connect_get_namespaced_pod_exec, name=name, namespace='default', command=['cat', path], stderr=True, stdin=False, stdout=True, tty=False) except ApiException: assert False assert result.rstrip('\n') == content,\ 'unexpected data in {}: expected "{}", got "{}"'.format( path, content, result ) utils.retry(_check_file_content, times=10, wait=2, name='checking content of {} on Pod {}'.format(path, name))
def utils_pod(k8s_client, utils_image): # Create the Pod manifest_file = os.path.join( os.path.realpath(os.path.dirname(__file__)), "files", "utils.yaml" ) with open(manifest_file, encoding='utf-8') as fd: manifest = yaml.safe_load(fd) manifest["spec"]["containers"][0]["image"] = utils_image pod_name = manifest["metadata"]["name"] k8s_client.create_namespaced_pod(body=manifest, namespace="default") # Wait for the Pod to be ready utils.retry( kube_utils.check_pod_status( k8s_client, name=pod_name, namespace="default", state="Running" ), times=10, wait=5, name="wait for Pod '{}'".format(pod_name), ) yield pod_name # Clean-up resources k8s_client.delete_namespaced_pod( name=pod_name, namespace="default", body=client.V1DeleteOptions( grace_period_seconds=0, # Force deletion instantly ), )
def busybox_pod(k8s_client): # Create the busybox pod pod_manifest = os.path.join(os.path.realpath(os.path.dirname(__file__)), "files", "busybox.yaml") with open(pod_manifest, encoding='utf-8') as pod_fd: pod_manifest_content = yaml.safe_load(pod_fd) k8s_client.create_namespaced_pod(body=pod_manifest_content, namespace="default") # Wait for the busybox to be ready utils.retry( kube_utils.wait_for_pod(k8s_client, name="busybox", namespace="default", state="Running"), times=10, wait=5, name="wait for Pod 'busybox'", ) yield "busybox" # Clean-up resources k8s_client.delete_namespaced_pod( name="busybox", namespace="default", body=client.V1DeleteOptions(), )
def pod_has_metrics(label, namespace, k8s_apiclient): def _pod_has_metrics(): result = k8s_apiclient.call_api( resource_path='/apis/metrics.k8s.io/v1beta1/' 'namespaces/{namespace}/pods', method='GET', response_type=object, path_params={ 'namespace': namespace, }, query_params=[ ('labelSelector', label), ], _return_http_data_only=True, ) assert result['apiVersion'] == 'metrics.k8s.io/v1beta1' assert result['kind'] == 'PodMetricsList' assert result['items'] != [] assert result['items'][0]['containers'] != [] assert result['items'][0]['containers'][0]['usage']['cpu'] assert result['items'][0]['containers'][0]['usage']['memory'] # Metrics are only available after a while (by design) utils.retry(_pod_has_metrics, times=60, wait=3)
def check_exec(host, command, label, namespace): # Just in case something is not ready yet, we make sure we can find # candidates before trying further def _wait_for_pods(): assert len(kube_utils.get_pods(host, label, namespace)) > 0 utils.retry(_wait_for_pods, times=10, wait=3, name="wait for pod labeled '{}'".format(label)) candidates = kube_utils.get_pods(host, label, namespace) assert len(candidates) == 1, ( "Expected only one Pod with label {l}, found {f}").format( l=label, f=len(candidates)) pod = candidates[0] with host.sudo(): host.check_output( 'kubectl --kubeconfig=/etc/kubernetes/admin.conf ' 'exec --namespace %s %s %s', namespace, pod['metadata']['name'], command, )
def wait_for_deletion(self, name): """Wait for the object to disappear.""" def _check_absence(): assert self.get(name) is None,\ '{} {} still exist'.format(self._kind, name) utils.retry(_check_absence, times=self._count, wait=self._delay, name='checking the absence of {} {}'.format( self._kind, name))
def check_static_pod_changed(host, hostname, k8s_client, static_pod_id): fullname = "{}-{}".format(DEFAULT_POD_NAME, hostname) utils.retry( kube_utils.wait_for_pod(k8s_client, fullname), times=10, wait=5, name="wait for Pod '{}'".format(fullname), ) pod = k8s_client.read_namespaced_pod(name=fullname, namespace="default") assert pod.metadata.uid != static_pod_id
def apiservice_exists(host, name, k8s_apiclient, request): client = kubernetes.client.ApiregistrationV1Api(api_client=k8s_apiclient) def _check_object_exists(): try: _ = client.read_api_service(name) except ApiException as err: if err.status == 404: raise AssertionError('APIService not yet created') raise utils.retry(_check_object_exists, times=20, wait=3)
def count_running_pods(host, min_pods_count, label): def _check_pods_count(): pods = kube_utils.get_pods( host, label, namespace="kube-system", status_phase="Running", ) assert len(pods) >= min_pods_count utils.retry(_check_pods_count, times=10, wait=3)
def check_pv_size(name, size, pv_client): def _check_pv_size(): pv = pv_client.get(name) assert pv is not None, 'PersistentVolume {} not found'.format(name) assert pv.spec.capacity['storage'] == size, \ 'Unexpected PersistentVolume size: expected {}, got {}'.format( size, pv.spec.capacity['storage'] ) utils.retry(_check_pv_size, times=10, wait=2, name='checking size of PersistentVolume {}'.format(name))
def create_with_volume(self, volume_name, command): """Create a pod using the specified volume.""" binary, *args = ast.literal_eval(command) body = POD_TEMPLATE.format(volume_name=volume_name, image_name=self._image, command=json.dumps(binary), args=json.dumps(args)) self.create_from_yaml(body) # Wait for the Pod to be up and running. pod_name = '{}-pod'.format(volume_name) utils.retry(kube_utils.check_pod_status(self._client, pod_name), times=self._count, wait=self._delay, name="wait for pod {}".format(pod_name))
def check_pv_label(name, key, value, pv_client): def _check_pv_label(): pv = pv_client.get(name) assert pv is not None, 'PersistentVolume {} not found'.format(name) labels = pv.metadata.labels assert key in labels, 'Label {} is missing'.format(key) assert labels[key] == value,\ 'Unexpected value for label {}: expected {}, got {}'.format( key, value, labels[key] ) utils.retry(_check_pv_label, times=10, wait=2, name='checking label of PersistentVolume {}'.format(name))
def assert_app_is_up(self, appname=None, msg='Hello World!'): if appname is None: appname = self.appname out, _ = tsuru.app_info('-a', appname) addr = re.search(r'Address: (.*?)\n', out).group(1) out, _ = retry(shell.curl, '-fsSL', addr) self.assertEqual(out, msg)
def check_deletion_marker(self, name): def _check_deletion_marker(): obj = self.get(name) assert obj is not None, '{} {} not found'.format(self._kind, name) if isinstance(obj, dict): tstamp = obj['metadata'].get('deletionTimestamp') else: tstamp = obj.metadata.deletion_timestamp assert tstamp is not None,\ '{} {} is not marked for deletion'.format(self._kind, name) utils.retry(_check_deletion_marker, times=self._count, wait=self._delay, name='checking that {} {} is marked for deletion'.format( self._kind, name))
def check_volume_status(context, name, status, volume_client): def _check_volume_status(): volume = volume_client.get(name) assert volume is not None, 'Volume {} not found'.format(name) context[name] = volume try: phase = VolumeClient.compute_phase(volume['status']) assert phase == status,\ 'Unexpected status: expected {}, got {}'.format(status, phase) except KeyError: assert status == 'Unknown', \ 'Unexpected status: expected {}, got none'.format(status) utils.retry(_check_volume_status, times=30, wait=2, name='checking status of Volume {}'.format(name))
def check_volume_error(context, name, code, pattern, volume_client): def _check_error(): volume = volume_client.get(name) assert volume is not None, 'Volume {} not found'.format(name) context[name] = volume status = volume.get('status') assert status is not None, 'no status for volume {}'.format(name) phase = VolumeClient.compute_phase(status) errcode, errmsg = VolumeClient.get_error(status) assert phase == 'Failed',\ 'Unexpected status: expected Failed, got {}'.format(status, phase) assert errcode == code,\ 'Unexpected error code: expected {}, got {}'.format(code, errcode) assert re.search(pattern, errmsg) is not None,\ "error message `{}` doesn't match `{}`".format(errmsg, pattern) utils.retry(_check_error, times=30, wait=2, name='checking error for Volume {}'.format(name))
def apiservice_condition_met(name, condition, k8s_apiclient): client = kubernetes.client.ApiregistrationV1Api(api_client=k8s_apiclient) def _check_object_exists(): try: svc = client.read_api_service(name) ok = False for cond in svc.status.conditions: if cond.type == condition: assert cond.status == 'True', \ '{} condition is True'.format(condition) ok = True assert ok, '{} condition not found'.format(condition) except ApiException as err: if err.status == 404: raise AssertionError('APIService not yet created') raise utils.retry(_check_object_exists, times=20, wait=3)
def check_job_health(host, job, namespace, health): def _wait_job_status(): response = _query_prometheus_api(host, 'targets') active_targets = response.json()['data']['activeTargets'] job_found = False for target in active_targets: if target['labels']['job'] == job and \ target['labels']['namespace'] == namespace: assert target['health'] == health, target['lastError'] job_found = True assert job_found, 'Unable to find {} in Prometheus targets'.format(job) # Here we do a lot of retries because some pods can be really slow to start # e.g. kube-state-metrics utils.retry(_wait_job_status, times=30, wait=3, name="wait for job '{}' in namespace '{}' being '{}'".format( job, namespace, health))
def _check_pods_status(k8s_client, expected_status, ssh_config, namespace=None, label=None): # Helper to use retry utils def _wait_for_status(): pods = kube_utils.get_pods( k8s_client, ssh_config, label, namespace=namespace ) assert pods for pod in pods: # If really not ready, status may not have been pushed yet. if pod.status.conditions is None: assert expected_status == 'NotReady' continue for condition in pod.status.conditions: if condition.type == 'Ready': break assert kube_utils.MAP_STATUS[condition.status] == expected_status return pods name = "wait for pods" if namespace: name += " in namespace {}".format(namespace) else: name += " in all namespaces" if label: name += " with label '{}'".format(label) # Wait for pod to be in the correct state utils.retry( _wait_for_status, times=12, wait=5, name=name )
def check_static_pod_changed(host, hostname, k8s_client, static_pod_id): fullname = "{}-{}".format(DEFAULT_POD_NAME, hostname) wait_for_pod = kube_utils.check_pod_status( k8s_client, name=fullname, namespace="default", ) def wait_for_pod_reloaded(): pod = wait_for_pod() assert pod.metadata.uid != static_pod_id utils.retry( wait_for_pod_reloaded, times=12, wait=5, name="wait for Pod '{}' to be reloaded".format(fullname), ) pod = k8s_client.read_namespaced_pod(name=fullname, namespace="default") assert pod.metadata.uid != static_pod_id
def check_node_status(ssh_config, k8s_client, hostname, expected_status): """Check if the given node has the expected status.""" node_name = utils.resolve_hostname(hostname, ssh_config) def _check_node_status(): try: status = k8s_client.read_node_status(node_name).status except k8s.client.rest.ApiException as exn: raise AssertionError(exn) # If really not ready, status may not have been pushed yet. if status.conditions is None: assert expected_status == 'NotReady' return for condition in status.conditions: if condition.type == 'Ready': break assert kube_utils.MAP_STATUS[condition.status] == expected_status utils.retry( _check_node_status, times=10, wait=5, name="check node '{}' status".format(node_name) )
def check_exec(request, host, k8s_client, command, label, namespace): ssh_config = request.config.getoption('--ssh-config') # Just in case something is not ready yet, we make sure we can find # candidates before trying further def _wait_for_pods(): pods = kube_utils.get_pods( k8s_client, ssh_config, label, namespace=namespace ) assert len(pods) > 0 utils.retry( _wait_for_pods, times=10, wait=3, name="wait for pod labeled '{}'".format(label) ) candidates = kube_utils.get_pods( k8s_client, ssh_config, label, namespace=namespace ) assert len(candidates) == 1, ( "Expected only one Pod with label {l}, found {f}" ).format(l=label, f=len(candidates)) pod = candidates[0] with host.sudo(): host.check_output( 'kubectl --kubeconfig=/etc/kubernetes/admin.conf ' 'exec --namespace %s %s %s', namespace, pod.metadata.name, command, )
def count_running_pods(request, k8s_client, pods_count, label, namespace, node): ssh_config = request.config.getoption('--ssh-config') def _check_pods_count(): pods = kube_utils.get_pods( k8s_client, ssh_config, label, node, namespace=namespace, state="Running", ) assert len(pods) == pods_count error_msg = ( "There is not exactly '{count}' pod(s) labeled '{label}' running " "in namespace '{namespace}'".format(count=pods_count, label=label, namespace=namespace)) if node: error_msg += "on node '{node}'".format(node=node) utils.retry(_check_pods_count, times=20, wait=3, error_msg=error_msg)
def tearDownClass(cls): if getattr(cls, 'appname', None): retry(tsuru.app_remove, '-a', cls.appname, '-y', count=1, ignore=r'.*not found.*') if getattr(cls, 'teamname', None): retry(tsuru.team_remove, cls.teamname, stdin='y', count=10, ignore=r'.*not found.*') if getattr(cls, 'username', None) and not os.environ.get('TSURU_TOKEN'): retry(tsuru.user_remove, stdin='y', count=1, ignore=r'.*not found.*')
def reset_user(cls): if not getattr(cls, 'username', None): cls.username = '******'.format(random_name()) cls.password = random_name() cls.appname = random_name('integration-test-app-') cls.teamname = random_name('integration-test-team-') cls.keyname = random_name('integration-test-key-') if os.environ.get('TSURU_TOKEN'): retry(tsuru.app_remove, '-a', cls.appname, '-y', count=10, ignore=r'.*not found.*') retry(tsuru.team_remove, cls.teamname, stdin='y', count=10, ignore=r'.*not found.*') tsuru.team_create(cls.teamname) return try: tsuru.login(cls.username, stdin=cls.password) except: pass else: retry(tsuru.app_remove, '-a', cls.appname, '-y', count=10, ignore=r'.*not found.*') retry(tsuru.key_remove, cls.keyname, '-y', count=1, ignore=r'.*not found.*') retry(tsuru.team_remove, cls.teamname, stdin='y', count=10, ignore=r'.*not found.*') retry(tsuru.user_remove, stdin='y', count=1, ignore=r'.*not found.*') tsuru.user_create(cls.username, stdin=cls.password + '\n' + cls.password) tsuru.login(cls.username, stdin=cls.password) tsuru.team_create(cls.teamname)
def set_up_static_pod(host, hostname, k8s_client, utils_image, transient_files): manifest_path = str(MANIFESTS_PATH / "{}.yaml".format(DEFAULT_POD_NAME)) with host.sudo(): if host.file(manifest_path).exists: pytest.fail("Cannot set up static Pod with manifest at path '{}': " "already exists".format(manifest_path)) # A sample config file for the static Pod config_path = "/tmp/{}.conf".format(DEFAULT_POD_NAME) write_config = utils.write_string(host, config_path, '{"hello": "world"}') assert write_config.rc == 0, ( "Failed to write static Pod config file at '{}': {}").format( config_path, write_config.stderr) transient_files.append(config_path) # The manifest template to use with Salt manifest_template = SOURCE_TEMPLATE.read_text(encoding="utf-8") manifest = string.Template(manifest_template).substitute( name=DEFAULT_POD_NAME, image=utils_image, config_path=config_path, ) with host.sudo(): host.run_test("mkdir -p %s", str(TEMPLATES_PATH)) template_path = str(TEMPLATES_PATH / "{}.yaml.j2".format(DEFAULT_POD_NAME)) write_template = utils.write_string(host, template_path, manifest) assert write_template.rc == 0, ( "Failed to create static Pod manifest template '{}': {}").format( template_path, write_template.stderr) transient_files.append(template_path) # Use Salt to generate the effective static Pod manifest _manage_static_pod(host, manifest_path, template_path, config_path) assert host.file(manifest_path).exists, ( "Something went wrong: " "static Pod manifest could not be found after set-up") # We want to remove the manifest before the config file, since kubelet # may try to mount it and, when not found, create a directory (bug). # See: https://github.com/kubernetes/kubernetes/issues/65825 transient_files.insert(0, manifest_path) fullname = "{}-{}".format(DEFAULT_POD_NAME, hostname) utils.retry( kube_utils.check_pod_status(k8s_client, fullname), times=10, wait=5, name="wait for Pod '{}'".format(fullname), ) pod = k8s_client.read_namespaced_pod(name=fullname, namespace="default") return pod.metadata.uid