def test_urlpath_err(self):
"""Test various error scenarios."""
# Valid version but invalid resource kind or invalid namespace spelling.
for version in SUPPORTED_VERSIONS:
cfg = Config("url", "token", "ca_cert", "client_cert", version)
# Invalid resource kind.
assert k8s.urlpath(cfg, "fooresource", "ns") == (None, True)
# Namespace names must be all lower case (K8s imposes this)...
assert k8s.urlpath(cfg, "Deployment", "namEspACe") == (None, True)
# Invalid version.
cfg = Config("url", "token", "ca_cert", "client_cert", "invalid")
assert k8s.urlpath(cfg, "Deployment", "valid-ns") == (None, True)
def test_urlpath_ok(self):
"""Must work for all supported K8s versions and resources."""
for version in SUPPORTED_VERSIONS:
cfg = Config("url", "token", "ca_cert", "client_cert", version)
for kind in SUPPORTED_KINDS:
for ns in (None, "foo-namespace"):
path, err = k8s.urlpath(cfg, kind, ns)
# Verify.
assert err is False
assert isinstance(path, str)
def test_make_patch_ok(self):
"""Compute patch between two manifests.
This test function first verifies that the patch between two identical
manifests is empty. The second used two manifests that have different
labels. This must produce two patch operations, one to remove the old
label and one to add the new ones.
"""
config = k8s.Config("url", "token", "ca_cert", "client_cert", "1.10")
# Two valid manifests.
kind, namespace, name = "Deployment", "namespace", "name"
srv = make_manifest(kind, namespace, name)
loc = make_manifest(kind, namespace, name)
srv["metadata"]["labels"] = {"old": "old"}
loc["metadata"]["labels"] = {"new": "new"}
# The Patch between two identical manifests must be a No-Op.
expected = JsonPatch(
url=urlpath(config, kind, namespace)[0] + f"/{name}",
ops=[],
)
assert square.make_patch(config, loc, loc) == (expected, False)
# The patch between `srv` and `loc` must remove the old label and add
# the new one.
expected = JsonPatch(url=urlpath(config, kind, namespace)[0] +
f"/{name}",
ops=[{
'op': 'remove',
'path': '/metadata/labels/old'
}, {
'op': 'add',
'path': '/metadata/labels/new',
'value': 'new'
}])
assert square.make_patch(config, loc, srv) == (expected, False)
def test_make_patch_empty(self):
"""Basic test: compute patch between two identical resources."""
# Setup.
kind, ns, name = 'Deployment', 'ns', 'foo'
config = k8s.Config("url", "token", "ca_cert", "client_cert", "1.10")
# PATCH URLs require the resource name at the end of the request path.
url = urlpath(config, kind, ns)[0] + f'/{name}'
# The patch must be empty for identical manifests.
loc = srv = make_manifest(kind, ns, name)
data, err = square.make_patch(config, loc, srv)
assert (data, err) == (JsonPatch(url, []), False)
assert isinstance(data, JsonPatch)
def test_compile_plan_patch_no_diff(self):
"""Test a plan that patches all resources.
To do this, the local and server resources are identical. As a
result, the returned plan must nominate all manifests for patching, and
none to create and delete.
"""
# Create vanilla `Config` instance.
config = k8s.Config("url", "token", "ca_cert", "client_cert", "1.10")
# Allocate arrays for the MetaManifests.
meta = [None] * 4
# Define two Namespace with 1 deployment each.
meta[0] = MetaManifest('v1', 'Namespace', None, 'ns1')
meta[1] = MetaManifest('v1', 'Deployment', 'ns1', 'res_0')
meta[2] = MetaManifest('v1', 'Namespace', None, 'ns2')
meta[3] = MetaManifest('v1', 'Deployment', 'ns2', 'res_1')
# Determine the K8s resource URLs for patching. Those URLs must contain
# the resource name as the last path element, eg "/api/v1/namespaces/ns1"
url = [
urlpath(config, _.kind, _.namespace)[0] + f"/{_.name}"
for _ in meta
]
# Local and server manifests are identical. The plan must therefore
# only nominate patches but nothing to create or delete.
loc_man = srv_man = {
meta[0]: make_manifest("Namespace", None, "ns1"),
meta[1]: make_manifest("Deployment", "ns1", "res_0"),
meta[2]: make_manifest("Namespace", None, "ns2"),
meta[3]: make_manifest("Deployment", "ns2", "res_1"),
}
expected = DeploymentPlan(create=[],
patch=[
DeltaPatch(meta[0], "",
JsonPatch(url[0], [])),
DeltaPatch(meta[1], "",
JsonPatch(url[1], [])),
DeltaPatch(meta[2], "",
JsonPatch(url[2], [])),
DeltaPatch(meta[3], "",
JsonPatch(url[3], [])),
],
delete=[])
assert square.compile_plan(config, loc_man,
srv_man) == (expected, False)
def test_make_patch_special(self):
"""Namespace, ClusterRole(Bindings) etc are special.
What makes them special is that they exist outside namespaces.
Therefore, they will/must not contain a `metadata.Namespace` attribute
and require special treatment in `make_patch`.
"""
# Generic fixtures; values are irrelevant.
config = types.SimpleNamespace(url='http://examples.com/',
version="1.10")
name = "foo"
for kind in ["Namespace", "ClusterRole"]:
# Determine the resource path so we can verify it later.
url = urlpath(config, kind, None)[0] + f'/{name}'
# The patch between two identical manifests must be empty but valid.
loc = srv = make_manifest(kind, None, name)
assert square.make_patch(config, loc, srv) == ((url, []), False)
# Create two almost identical manifests, except the second one has
# the forbidden `metadata.namespace` attribute. This must fail.
loc = make_manifest(kind, None, name)
srv = copy.deepcopy(loc)
loc['metadata']['namespace'] = 'foo'
data, err = square.make_patch(config, loc, srv)
assert data is None and err is not None
# Create two almost identical manifests, except the second one has
# different `metadata.labels`. This must succeed.
loc = make_manifest(kind, None, name)
srv = copy.deepcopy(loc)
loc['metadata']['labels'] = {"key": "value"}
data, err = square.make_patch(config, loc, srv)
assert err is False and len(data) > 0
def compile_plan(
config: Config,
local: ServerManifests,
server: ServerManifests) -> Tuple[Optional[DeploymentPlan], bool]:
"""Return the `DeploymentPlan` to transition K8s to state of `local`.
The deployment plan is a named tuple. It specifies which resources to
create, patch and delete to ensure that the state of K8s matches that
specified in `local`.
Inputs:
config: Config
local: ServerManifests
Should be output from `load_manifest` or `load`.
server: ServerManifests
Should be output from `manio.download`.
Returns:
DeploymentPlan
"""
# Partition the set of meta manifests into create/delete/patch groups.
plan, err = partition_manifests(local, server)
if err or not plan:
return (None, True)
# Sanity check: the resources to patch *must* exist in both local and
# server manifests. This is a bug if not.
assert set(plan.patch).issubset(set(local.keys()))
assert set(plan.patch).issubset(set(server.keys()))
# Compile the Deltas to create the missing resources.
create = []
for delta in plan.create:
url, err = k8s.urlpath(config, delta.kind, namespace=delta.namespace)
if err:
return (None, True)
create.append(DeltaCreate(delta, url, local[delta]))
# Compile the Deltas to delete the excess resources. Every DELETE request
# will have to pass along a `DeleteOptions` manifest (see below).
del_opts = {
"apiVersion": "v1",
"kind": "DeleteOptions",
"gracePeriodSeconds": 0,
"orphanDependents": False,
}
delete = []
for meta in plan.delete:
# Resource URL.
url, err = k8s.urlpath(config, meta.kind, namespace=meta.namespace)
if err:
return (None, True)
# DELETE requests must specify the resource name in the path.
url = f"{url}/{meta.name}"
# Assemble the meta and add it to the list.
delete.append(DeltaDelete(meta, url, del_opts.copy()))
# Iterate over each manifest that needs patching and determine the
# necessary JSON Patch to transition K8s into the state specified in the
# local manifests.
patches = []
for meta in plan.patch:
# Compute textual diff (only useful for the user to study the diff).
diff_str, err = manio.diff(config, local[meta], server[meta])
if err:
return (None, True)
# Compute the JSON patch that will match K8s to the local manifest.
patch, err = make_patch(config, local[meta], server[meta])
if err:
return (None, True)
patches.append(DeltaPatch(meta, diff_str, patch))
# Assemble and return the deployment plan.
return (DeploymentPlan(create, patches, delete), False)
def make_patch(
config: Config,
local: ServerManifests,
server: ServerManifests) -> Tuple[Optional[JsonPatch], bool]:
"""Return JSON patch to transition `server` to `local`.
Inputs:
local: LocalManifests
Usually the dictionary keys returned by `load_manifest`.
server: ServerManifests
Usually the dictionary keys returned by `manio.download`.
Returns:
Patch: the JSON patch and human readable diff in a `Patch` tuple.
"""
# Reduce local and server manifests to salient fields (ie apiVersion, kind,
# metadata and spec). Abort on error.
loc, err1 = manio.strip(config, local)
srv, err2 = manio.strip(config, server)
if err1 or err2 or loc is None or srv is None:
return (None, True)
# Log the manifest info for which we will try to compute a patch.
man_id = f"{loc.kind.upper()}: {loc.metadata.name}/{loc.metadata.name}"
logit.debug(f"Making patch for {man_id}")
# Sanity checks: abort if the manifests do not specify the same resource.
try:
assert srv.apiVersion == loc.apiVersion
assert srv.kind == loc.kind
assert srv.metadata.name == loc.metadata.name
# Not all resources live in a namespace, for instance Namespaces,
# ClusterRoles, ClusterRoleBindings. Here we ensure that the namespace
# in the local and server manifest matches for those resources that
# have a namespace.
if srv.kind in {"Namespace", "ClusterRole", "ClusterRoleBinding"}:
namespace = None
else:
assert srv.metadata.namespace == loc.metadata.namespace
namespace = srv.metadata.namespace
except AssertionError:
# Log the invalid manifests and return with an error.
keys = ("apiVersion", "kind", "metadata")
loc_tmp = {k: loc[k] for k in keys}
srv_tmp = {k: srv[k] for k in keys}
logit.error(
"Cannot compute JSON patch for incompatible manifests. "
f"Local: <{loc_tmp}> Server: <{srv_tmp}>"
)
return (None, True)
# Determine the PATCH URL for the resource.
url, err = k8s.urlpath(config, srv.kind, namespace)
if err:
return (None, True)
full_url = f'{url}/{srv.metadata.name}'
# Compute JSON patch.
patch = jsonpatch.make_patch(srv, loc)
patch = json.loads(patch.to_string())
# Return the patch.
return (JsonPatch(full_url, patch), False)
def download(
config: Config, client, kinds: Iterable[str],
namespaces: Optional[Iterable[str]]
) -> Tuple[Optional[ServerManifests], bool]:
"""Download and return the specified resource `kinds`.
Set `namespace` to None to download from all namespaces.
Either returns all the data or an error, never partial results.
Inputs:
config: Config
client: `requests` session with correct K8s certificates.
kinds: Iterable
The resource kinds, eg ["Deployment", "Namespace"]
namespaces: Iterable
Only query those namespaces. Set to None to download from all.
Returns:
Dict[MetaManifest, dict]: the K8s manifests from K8s.
"""
# Output.
server_manifests = {}
# Ensure `namespaces` is always a list to avoid special casing below.
all_namespaces: Iterable[Optional[str]]
if namespaces is None:
all_namespaces = [None]
else:
all_namespaces = namespaces
del namespaces
# Download each resource type. Abort at the first error and return nothing.
for namespace in all_namespaces:
for kind in kinds:
try:
# Get the HTTP URL for the resource request.
url, err = k8s.urlpath(config, kind, namespace)
assert not err and url is not None
# Make HTTP request.
manifest_list, err = k8s.get(client, url)
assert not err and manifest_list is not None
# Parse the K8s List (eg DeploymentList, NamespaceList, ...) into a
# Dict[MetaManifest, dict] dictionary.
manifests, err = unpack_list(manifest_list)
assert not err and manifests is not None
# Drop all manifest fields except "apiVersion", "metadata" and "spec".
ret = {k: strip(config, man) for k, man in manifests.items()}
# Ensure `strip` worked for every manifest.
err = any((v[1] for v in ret.values()))
assert not err
# Unpack the stripped manifests from the `strip` response.
# The "if v[0] is not None" is to satisfy MyPy - we already
# know they are not None or otherwise the previous assert would
# have failed.
manifests = {
k: v[0]
for k, v in ret.items() if v[0] is not None
}
except AssertionError:
# Return nothing, even if we had downloaded other kinds already.
return (None, True)
else:
# Copy the manifests into the output dictionary.
server_manifests.update(manifests)
return (server_manifests, False)