def _patch(self, _) -> None:
"""Patch the Kubernetes service created by Juju to map the correct port.
Raises:
PatchFailed: if patching fails due to lack of permissions, or otherwise.
"""
if not self.charm.unit.is_leader():
return
client = Client()
try:
client.patch(Service,
self._app,
self.service,
patch_type=PatchType.MERGE)
except ApiError as e:
if e.status.code == 403:
logger.error(
"Kubernetes service patch failed: `juju trust` this application."
)
else:
logger.error("Kubernetes service patch failed: %s", str(e))
else:
logger.info("Kubernetes service '%s' patched successfully",
self._app)
def test_pod_log(client: lightkube.Client):
result = ['line1\n', 'line2\n', 'line3\n']
content = "".join(result)
respx.get("https://localhost:9443/api/v1/namespaces/default/pods/test/log").respond(content=content)
lines = list(client.log('test'))
assert lines == result
respx.get("https://localhost:9443/api/v1/namespaces/default/pods/test/log?follow=true").respond(
content=content)
lines = list(client.log('test', follow=True))
assert lines == result
respx.get("https://localhost:9443/api/v1/namespaces/default/pods/test/log?tailLines=3").respond(
content=content)
lines = list(client.log('test', tail_lines=3))
assert lines == result
respx.get("https://localhost:9443/api/v1/namespaces/default/pods/test/log?since=30×tamps=true").respond(
content=content)
lines = list(client.log('test', since=30, timestamps=True))
assert lines == result
respx.get("https://localhost:9443/api/v1/namespaces/default/pods/test/log?container=bla").respond(
content=content)
lines = list(client.log('test', container="bla"))
assert lines == result
def test_global_methods():
client = Client()
nodes = [node.metadata.name for node in client.list(Node)]
assert len(nodes) > 0
node = client.get(Node, name=nodes[0])
assert node.metadata.name == nodes[0]
assert node.metadata.labels['kubernetes.io/os'] == node.status.nodeInfo.operatingSystem
def test_get_global(client: lightkube.Client):
respx.get("https://localhost:9443/api/v1/nodes/n1").respond(json={'metadata': {'name': 'n1'}})
pod = client.get(Node, name="n1")
assert pod.metadata.name == 'n1'
# GET doesn't support all namespaces
with pytest.raises(ValueError):
client.get(Pod, name="xx", namespace=lightkube.ALL_NS)
def test_get_namespaced(client: lightkube.Client):
respx.get("https://localhost:9443/api/v1/namespaces/default/pods/xx").respond(json={'metadata': {'name': 'xx'}})
pod = client.get(Pod, name="xx")
assert pod.metadata.name == 'xx'
respx.get("https://localhost:9443/api/v1/namespaces/other/pods/xx").respond(json={'metadata': {'name': 'xy'}})
pod = client.get(Pod, name="xx", namespace="other")
assert pod.metadata.name == 'xy'
def test_wait_deleted(client: lightkube.Client):
base_url = "https://localhost:9443/api/v1/nodes?fieldSelector=metadata.name%3Dtest-node&watch=true"
respx.get(base_url).respond(content=make_wait_deleted())
respx.get(base_url + "&resourceVersion=1").respond(content=make_wait_deleted())
message = "nodes/test-node was unexpectedly deleted"
with pytest.raises(lightkube.core.exceptions.ObjectDeleted, match=message):
client.wait(Node, "test-node", for_conditions=["TestCondition"])
def test_list_namespaced(client: lightkube.Client):
resp = {'items':[{'metadata': {'name': 'xx'}}, {'metadata': {'name': 'yy'}}]}
respx.get("https://localhost:9443/api/v1/namespaces/default/pods").respond(json=resp)
pods = client.list(Pod)
assert [pod.metadata.name for pod in pods] == ['xx', 'yy']
respx.get("https://localhost:9443/api/v1/namespaces/other/pods?labelSelector=k%3Dv").respond(json=resp)
pods = client.list(Pod, namespace="other", labels={'k': 'v'})
assert [pod.metadata.name for pod in pods] == ['xx', 'yy']
def test_wait_failed(client: lightkube.Client):
base_url = "https://localhost:9443/api/v1/nodes?fieldSelector=metadata.name%3Dtest-node&watch=true"
respx.get(base_url).respond(content=make_wait_failed())
respx.get(base_url + "&resourceVersion=1").respond(content=make_wait_failed())
message = r"nodes/test-node has failure condition\(s\): TestCondition"
with pytest.raises(lightkube.core.exceptions.ConditionError, match=message):
client.wait(Node, "test-node", for_conditions=[], raise_for_conditions=["TestCondition"])
def test_wait_custom(client: lightkube.Client):
base_url = "https://localhost:9443/apis/custom.org/v1/customs?fieldSelector=metadata.name%3Dcustom-resource&watch=true"
Custom = create_global_resource(
group="custom.org", version="v1", kind="Custom", plural="customs"
)
respx.get(base_url).respond(content=make_wait_custom())
respx.get(base_url + "&resourceVersion=1").respond(content=make_wait_custom())
client.wait(Custom, "custom-resource", for_conditions=["TestCondition"])
def test_errors(client: lightkube.Client):
respx.get("https://localhost:9443/api/v1/namespaces/default/pods/xx").respond(content="Error", status_code=409)
respx.get("https://localhost:9443/api/v1/namespaces/default/pods/xx").respond(json={'message': 'got problems'},
status_code=409)
with pytest.raises(httpx.HTTPError):
client.get(Pod, name="xx")
with pytest.raises(lightkube.ApiError, match='got problems') as exc:
client.get(Pod, name="xx")
assert exc.value.status.message == 'got problems'
def test_pod_not_exist():
client = Client()
with pytest.raises(ApiError) as exc_info:
client.get(Pod, name='this-pod-is-not-found')
status = exc_info.value.status
assert status.code == 404
assert status.details.name == 'this-pod-is-not-found'
assert status.reason == 'NotFound'
assert status.status == 'Failure'
def test_namespaced_methods(obj_name):
client = Client()
config = ConfigMap(
metadata=ObjectMeta(name=obj_name, namespace='default'),
data={'key1': 'value1', 'key2': 'value2'}
)
# create
config = client.create(config)
try:
assert config.metadata.name == obj_name
assert config.data['key1'] == 'value1'
assert config.data['key2'] == 'value2'
# replace
config.data['key1'] = 'new value'
config = client.replace(config)
assert config.data['key1'] == 'new value'
assert config.data['key2'] == 'value2'
# patch with PatchType.STRATEGIC
patch = {'metadata': {'labels': {'app': 'xyz'}}}
config = client.patch(ConfigMap, name=obj_name, obj=patch)
assert config.metadata.labels['app'] == 'xyz'
# get
config2 = client.get(ConfigMap, name=obj_name)
assert config.metadata.creationTimestamp == config2.metadata.creationTimestamp
# list
configs = [config.metadata.name for config in client.list(ConfigMap)]
assert obj_name in configs
finally:
client.delete(ConfigMap, name=obj_name)
def test_list_global(client: lightkube.Client):
resp = {'items': [{'metadata': {'name': 'xx'}}, {'metadata': {'name': 'yy'}}]}
respx.get("https://localhost:9443/api/v1/nodes").respond(json=resp)
nodes = client.list(Node)
assert [node.metadata.name for node in nodes] == ['xx', 'yy']
respx.get("https://localhost:9443/api/v1/pods?fieldSelector=k%3Dx").respond(json=resp)
pods = client.list(Pod, namespace=lightkube.ALL_NS, fields={'k': 'x'})
assert [pod.metadata.name for pod in pods] == ['xx', 'yy']
# Binding doesn't support all namespaces
with pytest.raises(ValueError):
client.list(Binding, namespace=lightkube.ALL_NS)
def test_replace_namespaced(client: lightkube.Client):
req = respx.put("https://localhost:9443/api/v1/namespaces/default/pods/xy").respond(json={'metadata': {'name': 'xy'}})
pod = client.replace(Pod(metadata=ObjectMeta(name="xy")))
assert req.calls[0][0].read() == b'{"metadata": {"name": "xy"}}'
assert pod.metadata.name == 'xy'
respx.put("https://localhost:9443/api/v1/namespaces/other/pods/xz").respond(json={'metadata': {'name': 'xz'}})
pod = client.replace(Pod(metadata=ObjectMeta(name="xz")), namespace='other')
assert pod.metadata.name == 'xz'
# namespace inside object definition need to match with provided namespace parameter.
with pytest.raises(ValueError):
client.replace(Pod(metadata=ObjectMeta(name="xx", namespace='ns1')), namespace='ns2')
def test_apply_global(client: lightkube.Client):
req = respx.patch("https://localhost:9443/api/v1/nodes/xy?fieldManager=test").respond(
json={'metadata': {'name': 'xy'}})
node = client.apply(Node(metadata=ObjectMeta(name='xy')), field_manager='test')
assert node.metadata.name == 'xy'
assert req.calls[0][0].headers['Content-Type'] == "application/apply-patch+yaml"
# sub-resource + force
req = respx.patch("https://localhost:9443/api/v1/nodes/xx/status?fieldManager=a&force=true").respond(
json={'metadata': {'name': 'xx'}})
node = client.apply(Node.Status(), name='xx', field_manager='a', force=True)
assert node.metadata.name == 'xx'
assert req.calls[0][0].headers['Content-Type'] == "application/apply-patch+yaml"
def __init__(self, *args):
super().__init__(*args)
if not self.unit.is_leader():
# We can't do anything useful when not the leader, so do nothing.
self.model.unit.status = WaitingStatus("Waiting for leadership")
return
try:
self.interfaces = get_interfaces(self)
except NoVersionsListed as err:
self.model.unit.status = WaitingStatus(str(err))
return
except NoCompatibleVersions as err:
self.model.unit.status = BlockedStatus(str(err))
return
else:
self.model.unit.status = ActiveStatus()
self.log = logging.getLogger(__name__)
self.env = Environment(loader=FileSystemLoader('src'))
self._resource_handler = ResourceHandler(self.app.name,
self.model.name)
self.lightkube_client = Client(namespace=self.model.name,
field_manager="lightkube")
self._resource_files = [
"gateway.yaml.j2",
"auth_filter.yaml.j2",
"virtual_service.yaml.j2",
]
self.framework.observe(self.on.install, self.install)
self.framework.observe(self.on.remove, self.remove)
self.framework.observe(self.on.config_changed,
self.handle_default_gateway)
self.framework.observe(self.on["istio-pilot"].relation_changed,
self.send_info)
self.framework.observe(self.on['ingress'].relation_changed,
self.handle_ingress)
self.framework.observe(self.on['ingress'].relation_broken,
self.handle_ingress)
self.framework.observe(self.on['ingress'].relation_departed,
self.handle_ingress)
self.framework.observe(self.on['ingress-auth'].relation_changed,
self.handle_ingress_auth)
self.framework.observe(self.on['ingress-auth'].relation_departed,
self.handle_ingress_auth)
def test_patch_global(client: lightkube.Client):
req = respx.patch("https://localhost:9443/api/v1/nodes/xx").respond(json={'metadata': {'name': 'xx'}})
node = client.patch(Node, "xx", [{"op": "add", "path": "/metadata/labels/x", "value": "y"}],
patch_type=types.PatchType.JSON)
assert node.metadata.name == 'xx'
assert req.calls[0][0].headers['Content-Type'] == "application/json-patch+json"
# PatchType.APPLY + force
req = respx.patch("https://localhost:9443/api/v1/nodes/xy?fieldManager=test&force=true").respond(
json={'metadata': {'name': 'xy'}})
node = client.patch(Node, "xy", Pod(metadata=ObjectMeta(labels={'l': 'ok'})),
patch_type=types.PatchType.APPLY, field_manager='test', force=True)
assert node.metadata.name == 'xy'
assert req.calls[0][0].headers['Content-Type'] == "application/apply-patch+yaml"
async def test_seldon_deployment(ops_test: OpsTest):
namespace = ops_test.model_name
client = Client()
this_ns = client.get(res=Namespace, name=namespace)
this_ns.metadata.labels.update({"serving.kubeflow.org/inferenceservice": "enabled"})
client.patch(res=Namespace, name=this_ns.metadata.name, obj=this_ns)
SeldonDeployment = create_namespaced_resource(
group="machinelearning.seldon.io",
version="v1",
kind="seldondeployment",
plural="seldondeployments",
verbs=None,
)
with open("examples/serve-simple-v1.yaml") as f:
sdep = SeldonDeployment(yaml.safe_load(f.read()))
client.create(sdep, namespace=namespace)
for i in range(30):
dep = client.get(SeldonDeployment, "seldon-model", namespace=namespace)
state = dep.get("status", {}).get("state")
if state == "Available":
logger.info(f"SeldonDeployment status == {state}")
break
else:
logger.info(f"SeldonDeployment status == {state} (waiting for 'Available')")
time.sleep(5)
else:
pytest.fail("Waited too long for seldondeployment/seldon-model!")
service_name = "seldon-model-example-classifier"
service = client.get(Service, name=service_name, namespace=namespace)
service_ip = service.spec.clusterIP
service_port = next(p for p in service.spec.ports if p.name == "http").port
response = requests.post(
f"http://{service_ip}:{service_port}/predict",
json={
"data": {
"names": ["a", "b"],
"tensor": {"shape": [2, 2], "values": [0, 0, 1, 1]},
}
},
)
response.raise_for_status()
response = response.json()
assert response["data"]["names"] == ["proba"]
assert response["data"]["tensor"]["shape"] == [2, 1]
assert response["meta"] == {}
def _check_deployed_resources(self, manifest=None):
"""Check the status of deployed resources, returning True if ok else raising CheckFailedError
All abnormalities are captured in logs
Params:
manifest: (Optional) list of lightkube objects describing the entire application. If
omitted, will be computed using self.get_manifest()
"""
if manifest:
expected_resources = manifest
else:
expected_resources = self.get_manifest()
found_resources = [None] * len(expected_resources)
errors = []
client = Client()
self.logger.info("Checking for expected resources")
for i, resource in enumerate(expected_resources):
try:
found_resources[i] = client.get(
type(resource),
resource.metadata.name,
namespace=resource.metadata.namespace,
)
except ApiError:
errors.append(
f"Cannot find k8s object for metadata '{resource.metadata}'"
)
self.logger.info(
"Checking readiness of found StatefulSets/Deployments")
statefulsets_ok, statefulsets_errors = validate_statefulsets_and_deployments(
found_resources)
errors.extend(statefulsets_errors)
# Log any errors
for err in errors:
self.logger.info(err)
if len(errors) == 0:
return True
else:
raise CheckFailedError(
"Some Kubernetes resources missing/not ready. See logs for details",
WaitingStatus,
)
def __init__(self, *args):
super().__init__(*args)
if not self.unit.is_leader():
# We can't do anything useful when not the leader, so do nothing.
self.model.unit.status = WaitingStatus("Waiting for leadership")
return
try:
self.interfaces = get_interfaces(self)
except NoVersionsListed as err:
self.model.unit.status = WaitingStatus(str(err))
return
except NoCompatibleVersions as err:
self.model.unit.status = BlockedStatus(str(err))
return
else:
self.model.unit.status = ActiveStatus()
self.log = logging.getLogger(__name__)
# Every lightkube API call will use the model name as the namespace by default
self.lightkube_client = Client(namespace=self.model.name,
field_manager="lightkube")
self.framework.observe(self.on.start, self.start)
self.framework.observe(self.on["istio-pilot"].relation_changed,
self.start)
self.framework.observe(self.on.config_changed, self.start)
self.framework.observe(self.on.remove, self.remove)
def test_list_chunk_size(client: lightkube.Client):
resp = {'items': [{'metadata': {'name': 'xx'}}, {'metadata': {'name': 'yy'}}], 'metadata': {'continue': 'yes'}}
respx.get("https://localhost:9443/api/v1/namespaces/default/pods?limit=3").respond(json=resp)
resp = {'items': [{'metadata': {'name': 'zz'}}]}
respx.get("https://localhost:9443/api/v1/namespaces/default/pods?limit=3&continue=yes").respond(json=resp)
pods = client.list(Pod, chunk_size=3)
assert [pod.metadata.name for pod in pods] == ['xx', 'yy', 'zz']
def is_patched(self) -> bool:
"""Reports if the service patch has been applied.
Returns:
bool: A boolean indicating if the service patch has been applied.
"""
client = Client()
# Get the relevant service from the cluster
service = client.get(Service,
name=self._app,
namespace=self._namespace)
# Construct a list of expected ports, should the patch be applied
expected_ports = [(p.port, p.targetPort)
for p in self.service.spec.ports]
# Construct a list in the same manner, using the fetched service
fetched_ports = [(p.port, p.targetPort) for p in service.spec.ports]
return expected_ports == fetched_ports
def delete_all_from_yaml(yaml_file: str,
lightkube_client: lightkube.Client = None):
"""Deletes all k8s resources listed in a YAML file via lightkube.
Args:
yaml_file (str or Path): Either a string filename or a string of valid YAML. Will attempt
to open a filename at this path, failing back to interpreting the
string directly as YAML.
lightkube_client: Instantiated lightkube client or None
"""
yaml_text = _safe_load_file_to_text(yaml_file)
if lightkube_client is None:
lightkube_client = lightkube.Client()
for obj in codecs.load_all_yaml(yaml_text):
lightkube_client.delete(type(obj), obj.metadata.name)
def test_patching(obj_name):
client = Client()
service = Service(
metadata=ObjectMeta(name=obj_name),
spec=ServiceSpec(
ports=[ServicePort(name='a', port=80, targetPort=8080)],
selector={'app': 'not-existing'}
)
)
# create
client.create(service)
try:
# patch with PatchType.STRATEGIC
patch = {'spec': {'ports': [{'name': 'b', 'port':81, 'targetPort': 8081}]}}
service = client.patch(Service, name=obj_name, obj=patch)
assert len(service.spec.ports) == 2
assert {port.name for port in service.spec.ports} == {'a', 'b'}
# strategic - patch merge key: port
# we also try to send a Resource type for patching
patch = Service(spec=ServiceSpec(ports=[ServicePort(name='b', port=81, targetPort=8082)]))
service = client.patch(Service, name=obj_name, obj=patch)
assert len(service.spec.ports) == 2
for port in service.spec.ports:
if port.port == 81:
assert port.targetPort == 8082
# patch with PatchType.MERGE
# merge will replace the full list
patch = {'spec': {'ports': [{'name': 'b', 'port': 81, 'targetPort': 8081}]}}
service = client.patch(Service, name=obj_name, obj=patch, patch_type=PatchType.MERGE)
assert len(service.spec.ports) == 1
assert service.spec.ports[0].port == 81
# patch with PatchType.JSON
patch = [
{'op': 'add', 'path': '/spec/ports/-', 'value': {'name': 'a', 'port': 80, 'targetPort': 8080}}
]
service = client.patch(Service, name=obj_name, obj=patch, patch_type=PatchType.JSON)
assert len(service.spec.ports) == 2
assert service.spec.ports[1].port == 80
finally:
client.delete(Service, name=obj_name)
def test_watch_stop_iter(client: lightkube.Client):
respx.get("https://localhost:9443/api/v1/nodes?watch=true").respond(content=make_watch_list())
respx.get("https://localhost:9443/api/v1/nodes?watch=true&resourceVersion=1").respond(status_code=404)
i = None
for i, _ in enumerate(client.watch(Node, on_error=types.on_error_raise)):
break
assert i == 0
def test_pod_apis(obj_name):
client = Client()
# list kube-system namespace
pods = [pod.metadata.name for pod in client.list(Pod, namespace='kube-system')]
assert len(pods) > 0
assert any(name.startswith('metrics-server') for name in pods)
# create a pod
pod = client.create(create_pod(obj_name, "while true;do echo 'this is a test';sleep 5; done"))
try:
assert pod.metadata.name == obj_name
assert pod.metadata.namespace == client.namespace
assert pod.status.phase
wait_pod(client, pod)
# read pod logs
for l in client.log(obj_name, follow=True):
assert l == 'this is a test\n'
break
finally:
# delete the pod
client.delete(Pod, obj_name)
def test_watch_on_error(client: lightkube.Client):
respx.get("https://localhost:9443/api/v1/nodes?watch=true").respond(content=make_watch_list())
respx.get("https://localhost:9443/api/v1/nodes?watch=true&resourceVersion=1").respond(status_code=404)
i = None
for i, (op, node) in enumerate(client.watch(Node, on_error=types.on_error_stop)):
assert node.metadata.name == f'p{i}'
assert op == 'ADDED'
assert i == 9
def test_wait_success(client: lightkube.Client):
base_url = "https://localhost:9443/api/v1/nodes?fieldSelector=metadata.name%3Dtest-node&watch=true"
respx.get(base_url).respond(content=make_wait_success())
respx.get(base_url + "&resourceVersion=1").respond(content=make_wait_success())
node = client.wait(Node, "test-node", for_conditions=["TestCondition"])
assert node.to_dict()["metadata"]["name"] == "test-node"
def __init__(self, app_name, model_name):
"""A Lightkube API interface.
Args:
- app_name: name of the application
- model_name: name of the Juju model this charm is deployed to
"""
self.app_name = app_name
self.model_name = model_name
self.log = logging.getLogger(__name__)
# Every lightkube API call will use the model name as the namespace by default
self.lightkube_client = Client(namespace=self.model_name,
field_manager="lightkube")
self.env = Environment(loader=FileSystemLoader('src'))
def test_create_namespaced(client: lightkube.Client):
req = respx.post("https://localhost:9443/api/v1/namespaces/default/pods").respond(json={'metadata': {'name': 'xx'}})
pod = client.create(Pod(metadata=ObjectMeta(name="xx", labels={'l': 'ok'})))
assert req.calls[0][0].read() == b'{"metadata": {"labels": {"l": "ok"}, "name": "xx"}}'
assert pod.metadata.name == 'xx'
req2 = respx.post("https://localhost:9443/api/v1/namespaces/other/pods").respond(json={'metadata': {'name': 'yy'}})
pod = client.create(Pod(metadata=ObjectMeta(name="xx", labels={'l': 'ok'})), namespace='other')
assert pod.metadata.name == 'yy'
assert req2.calls[0][0].read() == b'{"metadata": {"labels": {"l": "ok"}, "name": "xx"}}'
respx.post("https://localhost:9443/api/v1/namespaces/ns2/pods").respond(
json={'metadata': {'name': 'yy'}})
pod = client.create(Pod(metadata=ObjectMeta(name="xx", labels={'l': 'ok'}, namespace='ns2')))
assert pod.metadata.name == 'yy'
# namespace inside object definition need to match with provided namespace parameter.
with pytest.raises(ValueError):
client.create(Pod(metadata=ObjectMeta(name="xx", namespace='ns1')), namespace='ns2')