def partition_manifests(
local: ServerManifests,
server: ServerManifests) -> Tuple[DeploymentPlanMeta, bool]:
"""Compile `{local,server}` into CREATE, PATCH and DELETE groups.
The returned deployment plan will contain *every* resource in
`local` and `server` *exactly once*. Their relative
order will also be preserved.
Create: all resources that exist in `local` but not in `server`.
Delete: all resources that exist in `server` but not in `local`.
Patch : all resources that exist in both and therefore *may* need patching.
Inputs:
local: Dict[MetaManifest:str]
Usually the dictionary keys returned by `load_manifest`.
server: Dict[MetaManifest:str]
Usually the dictionary keys returned by `manio.download`.
Returns:
DeploymentPlanMeta
"""
# Determine what needs adding, removing and patching to steer the K8s setup
# towards what `local` specifies.
meta_loc = set(local.keys())
meta_srv = set(server.keys())
create = meta_loc - meta_srv
patch = meta_loc.intersection(meta_srv)
delete = meta_srv - meta_loc
del meta_loc, meta_srv
# Convert the sets to list. Preserve the relative element ordering as it
# was in `{local_server}`.
create_l = [_ for _ in local if _ in create]
patch_l = [_ for _ in local if _ in patch]
delete_l = [_ for _ in server if _ in delete]
# Return the deployment plan.
plan = DeploymentPlanMeta(create_l, patch_l, delete_l)
return (plan, False)
def compile_plan(
config: Config,
k8sconfig: K8sConfig,
local: ServerManifests,
server: ServerManifests) -> Tuple[DeploymentPlan, bool]:
"""Return the `DeploymentPlan` to transition K8s to the `local` state.
The deployment plan is a named tuple. It specifies which resources to
create, patch and delete to ensure that the state of K8s matches that
specified in `local`.
Inputs:
config: Square configuration.
k8sconfig: K8sConfig
local: ServerManifests
Should be output from `load_manifest` or `load`.
server: ServerManifests
Should be output from `manio.download`.
Returns:
DeploymentPlan
"""
err_resp = (DeploymentPlan(tuple(), tuple(), tuple()), True)
# Abort unless all local manifests reference valid K8s resource kinds.
if any([k8s.resource(k8sconfig, meta)[1] for meta in local]):
return err_resp
# Replace the server resources fetched from the K8s preferred endpoint with
# those fetched from the endpoint used by the local manifest.
server, err = match_api_version(k8sconfig, local, server)
assert not err
# Apply the filters to all local and server manifests before we compute patches.
server = {
meta: manio.strip(k8sconfig, man, config.filters)
for meta, man in server.items()
}
local = {
meta: manio.strip(k8sconfig, man, config.filters)
for meta, man in local.items()
}
# Abort if any of the manifests could not be stripped.
err_srv = {_[2] for _ in server.values()}
err_loc = {_[2] for _ in local.values()}
if True in err_srv or True in err_loc:
logit.error("Could not strip all manifests.")
return err_resp
# Unpack the stripped manifests (first element in the tuple returned from
# `manio.strip`).
server = {k: dotdict.undo(v[0]) for k, v in server.items()}
local = {k: dotdict.undo(v[0]) for k, v in local.items()}
# Partition the set of meta manifests into create/delete/patch groups.
plan, err = partition_manifests(local, server)
if err:
logit.error("Could not partition the manifests for the plan.")
return err_resp
# Sanity check: the resources to patch *must* exist in both local and
# server manifests. This is a bug if not.
assert set(plan.patch).issubset(set(local.keys()))
assert set(plan.patch).issubset(set(server.keys()))
# For later: every DELETE request will have to pass along a `DeleteOptions`
# manifest (see below).
del_opts = {
"apiVersion": "v1",
"kind": "DeleteOptions",
"gracePeriodSeconds": 0,
"orphanDependents": False,
}
# Compile the Deltas to create the missing resources.
create = []
for delta in plan.create:
# We only need the resource and namespace, not its name, because that
# is how the POST request to create a resource works in K8s.
# Ignore the error flag because the `strip` function we used above
# already ensured the resource exists.
resource, err = k8s.resource(k8sconfig, delta._replace(name=""))
assert not err
# Compile the Delta and add it to the list.
create.append(DeltaCreate(delta, resource.url, local[delta]))
# Compile the Deltas to delete the excess resources.
delete = []
for meta in plan.delete:
# Resource URL. Ignore the error flag because the `strip` function
# above already called `k8s.resource` and would have aborted on error.
resource, err = k8s.resource(k8sconfig, meta)
assert not err
# Compile the Delta and add it to the list.
delete.append(DeltaDelete(meta, resource.url, del_opts.copy()))
# Iterate over each manifest that needs patching and determine the
# necessary JSON Patch to transition K8s into the state specified in the
# local manifests.
patches = []
for meta in plan.patch:
# Compute textual diff (only useful for the user to study the diff).
diff_str, err = manio.diff(config, k8sconfig, local[meta], server[meta])
if err or diff_str is None:
logit.error(f"Could not compute the diff for <{meta}>.")
return err_resp
# Compute the JSON patch that will change the K8s state to match the
# one in the local files.
patch, err = make_patch(config, k8sconfig, local[meta], server[meta])
if err or patch is None:
logit.error(f"Could not compute the patch for <{meta}>")
return err_resp
# Append the patch to the list of patches, unless it is empty.
if len(patch.ops):
patches.append(DeltaPatch(meta, diff_str, patch))
# Assemble and return the deployment plan.
return (DeploymentPlan(create, patches, delete), False)
def align_serviceaccount(
local_manifests: ServerManifests,
server_manifests: ServerManifests) -> Tuple[ServerManifests, bool]:
"""Insert the token secret from `server_manifest` into `local_manifest`.
Every ServiceAccount (SA) has a "secrets" section that K8s automatically
populates when it creates the SA. The name contains a random hash, eg
"default-token-somerandomhash" for the default service account in every
namespace.
This makes it difficult to manage service accounts with Square because the
token is not known in advance. One would have to
square apply; square plan; square get serviceaccount
to sync this, and even that is not portable because the token will be
different on a new cluster.
To avoid this problem, this function will read the token secret that K8s
added (contained in `server_manifest`) and insert it into the
`local_manifest`. This will ensure that Square creates a plan that
leaves the token secret alone.
Inputs:
local_manifests: manifests from local files that the plan will use.
server_manifests: manifests from K8s
Returns:
Copy of `local_manifests` where all ServiceAccount token secrets match
those of the server.
"""
ReturnType = Tuple[Optional[str], List[Dict[str, str]], bool]
def _get_token(meta: MetaManifest, manifests: ServerManifests) -> ReturnType:
"""Return token secret from `manifest` as well as all other other secrets.
Example input manifest:
{
'apiVersion': v1,
'kind': ServiceAccount,
...
'secrets': [
{'name': 'some-secret'},
{'name': 'demoapp-token-abcde'},
{'name': 'other-secret'},
]
}
The output for this would be:
(
'demoapp-token-abcde',
[{'name': 'some-secret'}, {'name': 'other-secret'}],
False,
)
"""
# Do nothing if the ServiceAccount has no "secrets" - should be impossible.
try:
secrets_dict = manifests[meta]["secrets"]
except KeyError:
return (None, [], False)
# Find the ServiceAccount token name.
token_prefix = f"{meta.name}-token-"
secrets = [_["name"] for _ in secrets_dict]
token = [_ for _ in secrets if _.startswith(token_prefix)]
if len(token) == 0:
# No token - return the original secrets.
return (None, secrets_dict, False)
elif len(token) == 1:
# Expected case: return the token as well as the remaining secrets.
secrets = [{"name": _} for _ in secrets if _ != token[0]]
return (token[0], secrets, False)
else:
# Unexpected.
all_secrets = str.join(", ", list(sorted(token)))
logit.warning(
f"ServiceAccount <{meta.namespace}/{meta.name}>: "
f"found multiple token secrets in: `{all_secrets}`"
)
return (None, [], True)
# Avoid side effects.
local_manifests = copy.deepcopy(local_manifests)
# Find all ServiceAccount manifests that exist locally and on the cluster.
local_meta = {k for k in local_manifests if k.kind == "ServiceAccount"}
server_meta = set(server_manifests.keys()).intersection(local_meta)
# Iterate over all ServiceAccount manifests and insert the secret token
# from the cluster into the local manifest.
for meta in server_meta:
# Find the service account token in the local/cluster manifest.
loc_token, loc_secrets, err1 = _get_token(meta, local_manifests)
srv_token, srv_secrets, err2 = _get_token(meta, server_manifests)
# Ignore the manifest if there was an error. Typically this means the
# local or cluster manifest defined multiple service account secrets.
# If that happens then something is probably seriously wrong with the
# cluster.
if err1 or err2:
continue
# Server has no token - something is probably wrong with your cluster.
if srv_token is None:
logit.warning(
f"ServiceAccount {meta.namespace}/{meta.name} has no token secret"
)
continue
# This is the expected case: the local manifest does not specify
# the token but on the cluster it exists. In that case, add the
# token here.
if srv_token and not loc_token:
loc_secrets.append({"name": srv_token})
local_manifests[meta]["secrets"] = loc_secrets
return (local_manifests, False)