def check_logs(k8s_client):
pod_status = 'Running'
all_pods = k8s_client.list_pod_for_all_namespaces()
for pod in all_pods.items:
pod_name = pod.metadata.name
namespace = pod.metadata.namespace
for container in pod.spec.containers:
if container.name in SKIP_CONTAINERS:
continue
if pod.status.phase != pod_status:
# Wait for the Pod to be ready
utils.retry(
kube_utils.check_pod_status(
k8s_client,
name=pod_name,
namespace=namespace,
state=pod_status
),
times=12,
wait=5,
name="wait for Pod '{}'".format(pod_name),
)
logs = k8s_client.read_namespaced_pod_log(
pod_name,
namespace,
container=container.name
)
assert logs.strip(), (
"Couldn't find logs for container '{}' in Pod '{}' (status {})"
).format(container.name, pod_name, pod.status.phase)
def check_file_content_inside_pod(volume_name, path, content, k8s_client):
name = '{}-pod'.format(volume_name)
def _check_file_content():
try:
result = k8s.stream.stream(
k8s_client.connect_get_namespaced_pod_exec,
name=name,
namespace='default',
command=['cat', path],
stderr=True,
stdin=False,
stdout=True,
tty=False)
except ApiException:
assert False
assert result.rstrip('\n') == content,\
'unexpected data in {}: expected "{}", got "{}"'.format(
path, content, result
)
utils.retry(_check_file_content,
times=10,
wait=2,
name='checking content of {} on Pod {}'.format(path, name))
def utils_pod(k8s_client, utils_image):
# Create the Pod
manifest_file = os.path.join(
os.path.realpath(os.path.dirname(__file__)),
"files",
"utils.yaml"
)
with open(manifest_file, encoding='utf-8') as fd:
manifest = yaml.safe_load(fd)
manifest["spec"]["containers"][0]["image"] = utils_image
pod_name = manifest["metadata"]["name"]
k8s_client.create_namespaced_pod(body=manifest, namespace="default")
# Wait for the Pod to be ready
utils.retry(
kube_utils.check_pod_status(
k8s_client, name=pod_name, namespace="default", state="Running"
),
times=10,
wait=5,
name="wait for Pod '{}'".format(pod_name),
)
yield pod_name
# Clean-up resources
k8s_client.delete_namespaced_pod(
name=pod_name,
namespace="default",
body=client.V1DeleteOptions(
grace_period_seconds=0, # Force deletion instantly
),
)
def busybox_pod(k8s_client):
# Create the busybox pod
pod_manifest = os.path.join(os.path.realpath(os.path.dirname(__file__)),
"files", "busybox.yaml")
with open(pod_manifest, encoding='utf-8') as pod_fd:
pod_manifest_content = yaml.safe_load(pod_fd)
k8s_client.create_namespaced_pod(body=pod_manifest_content,
namespace="default")
# Wait for the busybox to be ready
utils.retry(
kube_utils.wait_for_pod(k8s_client,
name="busybox",
namespace="default",
state="Running"),
times=10,
wait=5,
name="wait for Pod 'busybox'",
)
yield "busybox"
# Clean-up resources
k8s_client.delete_namespaced_pod(
name="busybox",
namespace="default",
body=client.V1DeleteOptions(),
)
def pod_has_metrics(label, namespace, k8s_apiclient):
def _pod_has_metrics():
result = k8s_apiclient.call_api(
resource_path='/apis/metrics.k8s.io/v1beta1/'
'namespaces/{namespace}/pods',
method='GET',
response_type=object,
path_params={
'namespace': namespace,
},
query_params=[
('labelSelector', label),
],
_return_http_data_only=True,
)
assert result['apiVersion'] == 'metrics.k8s.io/v1beta1'
assert result['kind'] == 'PodMetricsList'
assert result['items'] != []
assert result['items'][0]['containers'] != []
assert result['items'][0]['containers'][0]['usage']['cpu']
assert result['items'][0]['containers'][0]['usage']['memory']
# Metrics are only available after a while (by design)
utils.retry(_pod_has_metrics, times=60, wait=3)
def check_exec(host, command, label, namespace):
# Just in case something is not ready yet, we make sure we can find
# candidates before trying further
def _wait_for_pods():
assert len(kube_utils.get_pods(host, label, namespace)) > 0
utils.retry(_wait_for_pods,
times=10,
wait=3,
name="wait for pod labeled '{}'".format(label))
candidates = kube_utils.get_pods(host, label, namespace)
assert len(candidates) == 1, (
"Expected only one Pod with label {l}, found {f}").format(
l=label, f=len(candidates))
pod = candidates[0]
with host.sudo():
host.check_output(
'kubectl --kubeconfig=/etc/kubernetes/admin.conf '
'exec --namespace %s %s %s',
namespace,
pod['metadata']['name'],
command,
)
def wait_for_deletion(self, name):
"""Wait for the object to disappear."""
def _check_absence():
assert self.get(name) is None,\
'{} {} still exist'.format(self._kind, name)
utils.retry(_check_absence,
times=self._count,
wait=self._delay,
name='checking the absence of {} {}'.format(
self._kind, name))
def check_static_pod_changed(host, hostname, k8s_client, static_pod_id):
fullname = "{}-{}".format(DEFAULT_POD_NAME, hostname)
utils.retry(
kube_utils.wait_for_pod(k8s_client, fullname),
times=10,
wait=5,
name="wait for Pod '{}'".format(fullname),
)
pod = k8s_client.read_namespaced_pod(name=fullname, namespace="default")
assert pod.metadata.uid != static_pod_id
def apiservice_exists(host, name, k8s_apiclient, request):
client = kubernetes.client.ApiregistrationV1Api(api_client=k8s_apiclient)
def _check_object_exists():
try:
_ = client.read_api_service(name)
except ApiException as err:
if err.status == 404:
raise AssertionError('APIService not yet created')
raise
utils.retry(_check_object_exists, times=20, wait=3)
def count_running_pods(host, min_pods_count, label):
def _check_pods_count():
pods = kube_utils.get_pods(
host,
label,
namespace="kube-system",
status_phase="Running",
)
assert len(pods) >= min_pods_count
utils.retry(_check_pods_count, times=10, wait=3)
def check_pv_size(name, size, pv_client):
def _check_pv_size():
pv = pv_client.get(name)
assert pv is not None, 'PersistentVolume {} not found'.format(name)
assert pv.spec.capacity['storage'] == size, \
'Unexpected PersistentVolume size: expected {}, got {}'.format(
size, pv.spec.capacity['storage']
)
utils.retry(_check_pv_size,
times=10,
wait=2,
name='checking size of PersistentVolume {}'.format(name))
def create_with_volume(self, volume_name, command):
"""Create a pod using the specified volume."""
binary, *args = ast.literal_eval(command)
body = POD_TEMPLATE.format(volume_name=volume_name,
image_name=self._image,
command=json.dumps(binary),
args=json.dumps(args))
self.create_from_yaml(body)
# Wait for the Pod to be up and running.
pod_name = '{}-pod'.format(volume_name)
utils.retry(kube_utils.check_pod_status(self._client, pod_name),
times=self._count,
wait=self._delay,
name="wait for pod {}".format(pod_name))
def check_pv_label(name, key, value, pv_client):
def _check_pv_label():
pv = pv_client.get(name)
assert pv is not None, 'PersistentVolume {} not found'.format(name)
labels = pv.metadata.labels
assert key in labels, 'Label {} is missing'.format(key)
assert labels[key] == value,\
'Unexpected value for label {}: expected {}, got {}'.format(
key, value, labels[key]
)
utils.retry(_check_pv_label,
times=10,
wait=2,
name='checking label of PersistentVolume {}'.format(name))
def assert_app_is_up(self, appname=None, msg='Hello World!'):
if appname is None:
appname = self.appname
out, _ = tsuru.app_info('-a', appname)
addr = re.search(r'Address: (.*?)\n', out).group(1)
out, _ = retry(shell.curl, '-fsSL', addr)
self.assertEqual(out, msg)
def check_deletion_marker(self, name):
def _check_deletion_marker():
obj = self.get(name)
assert obj is not None, '{} {} not found'.format(self._kind, name)
if isinstance(obj, dict):
tstamp = obj['metadata'].get('deletionTimestamp')
else:
tstamp = obj.metadata.deletion_timestamp
assert tstamp is not None,\
'{} {} is not marked for deletion'.format(self._kind, name)
utils.retry(_check_deletion_marker,
times=self._count,
wait=self._delay,
name='checking that {} {} is marked for deletion'.format(
self._kind, name))
def check_volume_status(context, name, status, volume_client):
def _check_volume_status():
volume = volume_client.get(name)
assert volume is not None, 'Volume {} not found'.format(name)
context[name] = volume
try:
phase = VolumeClient.compute_phase(volume['status'])
assert phase == status,\
'Unexpected status: expected {}, got {}'.format(status, phase)
except KeyError:
assert status == 'Unknown', \
'Unexpected status: expected {}, got none'.format(status)
utils.retry(_check_volume_status,
times=30,
wait=2,
name='checking status of Volume {}'.format(name))
def check_volume_error(context, name, code, pattern, volume_client):
def _check_error():
volume = volume_client.get(name)
assert volume is not None, 'Volume {} not found'.format(name)
context[name] = volume
status = volume.get('status')
assert status is not None, 'no status for volume {}'.format(name)
phase = VolumeClient.compute_phase(status)
errcode, errmsg = VolumeClient.get_error(status)
assert phase == 'Failed',\
'Unexpected status: expected Failed, got {}'.format(status, phase)
assert errcode == code,\
'Unexpected error code: expected {}, got {}'.format(code, errcode)
assert re.search(pattern, errmsg) is not None,\
"error message `{}` doesn't match `{}`".format(errmsg, pattern)
utils.retry(_check_error,
times=30,
wait=2,
name='checking error for Volume {}'.format(name))
def apiservice_condition_met(name, condition, k8s_apiclient):
client = kubernetes.client.ApiregistrationV1Api(api_client=k8s_apiclient)
def _check_object_exists():
try:
svc = client.read_api_service(name)
ok = False
for cond in svc.status.conditions:
if cond.type == condition:
assert cond.status == 'True', \
'{} condition is True'.format(condition)
ok = True
assert ok, '{} condition not found'.format(condition)
except ApiException as err:
if err.status == 404:
raise AssertionError('APIService not yet created')
raise
utils.retry(_check_object_exists, times=20, wait=3)
def check_job_health(host, job, namespace, health):
def _wait_job_status():
response = _query_prometheus_api(host, 'targets')
active_targets = response.json()['data']['activeTargets']
job_found = False
for target in active_targets:
if target['labels']['job'] == job and \
target['labels']['namespace'] == namespace:
assert target['health'] == health, target['lastError']
job_found = True
assert job_found, 'Unable to find {} in Prometheus targets'.format(job)
# Here we do a lot of retries because some pods can be really slow to start
# e.g. kube-state-metrics
utils.retry(_wait_job_status,
times=30,
wait=3,
name="wait for job '{}' in namespace '{}' being '{}'".format(
job, namespace, health))
def _check_pods_status(k8s_client, expected_status, ssh_config,
namespace=None, label=None):
# Helper to use retry utils
def _wait_for_status():
pods = kube_utils.get_pods(
k8s_client, ssh_config, label,
namespace=namespace
)
assert pods
for pod in pods:
# If really not ready, status may not have been pushed yet.
if pod.status.conditions is None:
assert expected_status == 'NotReady'
continue
for condition in pod.status.conditions:
if condition.type == 'Ready':
break
assert kube_utils.MAP_STATUS[condition.status] == expected_status
return pods
name = "wait for pods"
if namespace:
name += " in namespace {}".format(namespace)
else:
name += " in all namespaces"
if label:
name += " with label '{}'".format(label)
# Wait for pod to be in the correct state
utils.retry(
_wait_for_status,
times=12,
wait=5,
name=name
)
def check_static_pod_changed(host, hostname, k8s_client, static_pod_id):
fullname = "{}-{}".format(DEFAULT_POD_NAME, hostname)
wait_for_pod = kube_utils.check_pod_status(
k8s_client,
name=fullname,
namespace="default",
)
def wait_for_pod_reloaded():
pod = wait_for_pod()
assert pod.metadata.uid != static_pod_id
utils.retry(
wait_for_pod_reloaded,
times=12,
wait=5,
name="wait for Pod '{}' to be reloaded".format(fullname),
)
pod = k8s_client.read_namespaced_pod(name=fullname, namespace="default")
assert pod.metadata.uid != static_pod_id
def check_node_status(ssh_config, k8s_client, hostname, expected_status):
"""Check if the given node has the expected status."""
node_name = utils.resolve_hostname(hostname, ssh_config)
def _check_node_status():
try:
status = k8s_client.read_node_status(node_name).status
except k8s.client.rest.ApiException as exn:
raise AssertionError(exn)
# If really not ready, status may not have been pushed yet.
if status.conditions is None:
assert expected_status == 'NotReady'
return
for condition in status.conditions:
if condition.type == 'Ready':
break
assert kube_utils.MAP_STATUS[condition.status] == expected_status
utils.retry(
_check_node_status,
times=10, wait=5,
name="check node '{}' status".format(node_name)
)
def check_exec(request, host, k8s_client, command, label, namespace):
ssh_config = request.config.getoption('--ssh-config')
# Just in case something is not ready yet, we make sure we can find
# candidates before trying further
def _wait_for_pods():
pods = kube_utils.get_pods(
k8s_client, ssh_config, label, namespace=namespace
)
assert len(pods) > 0
utils.retry(
_wait_for_pods,
times=10,
wait=3,
name="wait for pod labeled '{}'".format(label)
)
candidates = kube_utils.get_pods(
k8s_client, ssh_config, label, namespace=namespace
)
assert len(candidates) == 1, (
"Expected only one Pod with label {l}, found {f}"
).format(l=label, f=len(candidates))
pod = candidates[0]
with host.sudo():
host.check_output(
'kubectl --kubeconfig=/etc/kubernetes/admin.conf '
'exec --namespace %s %s %s',
namespace,
pod.metadata.name,
command,
)
def count_running_pods(request, k8s_client, pods_count, label, namespace,
node):
ssh_config = request.config.getoption('--ssh-config')
def _check_pods_count():
pods = kube_utils.get_pods(
k8s_client,
ssh_config,
label,
node,
namespace=namespace,
state="Running",
)
assert len(pods) == pods_count
error_msg = (
"There is not exactly '{count}' pod(s) labeled '{label}' running "
"in namespace '{namespace}'".format(count=pods_count,
label=label,
namespace=namespace))
if node:
error_msg += "on node '{node}'".format(node=node)
utils.retry(_check_pods_count, times=20, wait=3, error_msg=error_msg)
def tearDownClass(cls):
if getattr(cls, 'appname', None):
retry(tsuru.app_remove,
'-a',
cls.appname,
'-y',
count=1,
ignore=r'.*not found.*')
if getattr(cls, 'teamname', None):
retry(tsuru.team_remove,
cls.teamname,
stdin='y',
count=10,
ignore=r'.*not found.*')
if getattr(cls, 'username',
None) and not os.environ.get('TSURU_TOKEN'):
retry(tsuru.user_remove,
stdin='y',
count=1,
ignore=r'.*not found.*')
def reset_user(cls):
if not getattr(cls, 'username', None):
cls.username = '******'.format(random_name())
cls.password = random_name()
cls.appname = random_name('integration-test-app-')
cls.teamname = random_name('integration-test-team-')
cls.keyname = random_name('integration-test-key-')
if os.environ.get('TSURU_TOKEN'):
retry(tsuru.app_remove,
'-a',
cls.appname,
'-y',
count=10,
ignore=r'.*not found.*')
retry(tsuru.team_remove,
cls.teamname,
stdin='y',
count=10,
ignore=r'.*not found.*')
tsuru.team_create(cls.teamname)
return
try:
tsuru.login(cls.username, stdin=cls.password)
except:
pass
else:
retry(tsuru.app_remove,
'-a',
cls.appname,
'-y',
count=10,
ignore=r'.*not found.*')
retry(tsuru.key_remove,
cls.keyname,
'-y',
count=1,
ignore=r'.*not found.*')
retry(tsuru.team_remove,
cls.teamname,
stdin='y',
count=10,
ignore=r'.*not found.*')
retry(tsuru.user_remove,
stdin='y',
count=1,
ignore=r'.*not found.*')
tsuru.user_create(cls.username,
stdin=cls.password + '\n' + cls.password)
tsuru.login(cls.username, stdin=cls.password)
tsuru.team_create(cls.teamname)
def set_up_static_pod(host, hostname, k8s_client, utils_image,
transient_files):
manifest_path = str(MANIFESTS_PATH / "{}.yaml".format(DEFAULT_POD_NAME))
with host.sudo():
if host.file(manifest_path).exists:
pytest.fail("Cannot set up static Pod with manifest at path '{}': "
"already exists".format(manifest_path))
# A sample config file for the static Pod
config_path = "/tmp/{}.conf".format(DEFAULT_POD_NAME)
write_config = utils.write_string(host, config_path, '{"hello": "world"}')
assert write_config.rc == 0, (
"Failed to write static Pod config file at '{}': {}").format(
config_path, write_config.stderr)
transient_files.append(config_path)
# The manifest template to use with Salt
manifest_template = SOURCE_TEMPLATE.read_text(encoding="utf-8")
manifest = string.Template(manifest_template).substitute(
name=DEFAULT_POD_NAME,
image=utils_image,
config_path=config_path,
)
with host.sudo():
host.run_test("mkdir -p %s", str(TEMPLATES_PATH))
template_path = str(TEMPLATES_PATH /
"{}.yaml.j2".format(DEFAULT_POD_NAME))
write_template = utils.write_string(host, template_path, manifest)
assert write_template.rc == 0, (
"Failed to create static Pod manifest template '{}': {}").format(
template_path, write_template.stderr)
transient_files.append(template_path)
# Use Salt to generate the effective static Pod manifest
_manage_static_pod(host, manifest_path, template_path, config_path)
assert host.file(manifest_path).exists, (
"Something went wrong: "
"static Pod manifest could not be found after set-up")
# We want to remove the manifest before the config file, since kubelet
# may try to mount it and, when not found, create a directory (bug).
# See: https://github.com/kubernetes/kubernetes/issues/65825
transient_files.insert(0, manifest_path)
fullname = "{}-{}".format(DEFAULT_POD_NAME, hostname)
utils.retry(
kube_utils.check_pod_status(k8s_client, fullname),
times=10,
wait=5,
name="wait for Pod '{}'".format(fullname),
)
pod = k8s_client.read_namespaced_pod(name=fullname, namespace="default")
return pod.metadata.uid
