def partition_manifests(
local: ServerManifests,
server: ServerManifests) -> Tuple[DeploymentPlanMeta, bool]:
"""Compile `{local,server}` into CREATE, PATCH and DELETE groups.
The returned deployment plan will contain *every* resource in
`local` and `server` *exactly once*. Their relative
order will also be preserved.
Create: all resources that exist in `local` but not in `server`.
Delete: all resources that exist in `server` but not in `local`.
Patch : all resources that exist in both and therefore *may* need patching.
Inputs:
local: Dict[MetaManifest:str]
Usually the dictionary keys returned by `load_manifest`.
server: Dict[MetaManifest:str]
Usually the dictionary keys returned by `manio.download`.
Returns:
DeploymentPlanMeta
"""
# Determine what needs adding, removing and patching to steer the K8s setup
# towards what `local` specifies.
meta_loc = set(local.keys())
meta_srv = set(server.keys())
create = meta_loc - meta_srv
patch = meta_loc.intersection(meta_srv)
delete = meta_srv - meta_loc
del meta_loc, meta_srv
# Convert the sets to list. Preserve the relative element ordering as it
# was in `{local_server}`.
create_l = [_ for _ in local if _ in create]
patch_l = [_ for _ in local if _ in patch]
delete_l = [_ for _ in server if _ in delete]
# Return the deployment plan.
plan = DeploymentPlanMeta(create_l, patch_l, delete_l)
return (plan, False)
def compile_plan(
config: Config,
k8sconfig: K8sConfig,
local: ServerManifests,
server: ServerManifests) -> Tuple[DeploymentPlan, bool]:
"""Return the `DeploymentPlan` to transition K8s to the `local` state.
The deployment plan is a named tuple. It specifies which resources to
create, patch and delete to ensure that the state of K8s matches that
specified in `local`.
Inputs:
config: Square configuration.
k8sconfig: K8sConfig
local: ServerManifests
Should be output from `load_manifest` or `load`.
server: ServerManifests
Should be output from `manio.download`.
Returns:
DeploymentPlan
"""
err_resp = (DeploymentPlan(tuple(), tuple(), tuple()), True)
# Abort unless all local manifests reference valid K8s resource kinds.
if any([k8s.resource(k8sconfig, meta)[1] for meta in local]):
return err_resp
# Replace the server resources fetched from the K8s preferred endpoint with
# those fetched from the endpoint used by the local manifest.
server, err = match_api_version(k8sconfig, local, server)
assert not err
# Apply the filters to all local and server manifests before we compute patches.
server = {
meta: manio.strip(k8sconfig, man, config.filters)
for meta, man in server.items()
}
local = {
meta: manio.strip(k8sconfig, man, config.filters)
for meta, man in local.items()
}
# Abort if any of the manifests could not be stripped.
err_srv = {_[2] for _ in server.values()}
err_loc = {_[2] for _ in local.values()}
if True in err_srv or True in err_loc:
logit.error("Could not strip all manifests.")
return err_resp
# Unpack the stripped manifests (first element in the tuple returned from
# `manio.strip`).
server = {k: dotdict.undo(v[0]) for k, v in server.items()}
local = {k: dotdict.undo(v[0]) for k, v in local.items()}
# Partition the set of meta manifests into create/delete/patch groups.
plan, err = partition_manifests(local, server)
if err:
logit.error("Could not partition the manifests for the plan.")
return err_resp
# Sanity check: the resources to patch *must* exist in both local and
# server manifests. This is a bug if not.
assert set(plan.patch).issubset(set(local.keys()))
assert set(plan.patch).issubset(set(server.keys()))
# For later: every DELETE request will have to pass along a `DeleteOptions`
# manifest (see below).
del_opts = {
"apiVersion": "v1",
"kind": "DeleteOptions",
"gracePeriodSeconds": 0,
"orphanDependents": False,
}
# Compile the Deltas to create the missing resources.
create = []
for delta in plan.create:
# We only need the resource and namespace, not its name, because that
# is how the POST request to create a resource works in K8s.
# Ignore the error flag because the `strip` function we used above
# already ensured the resource exists.
resource, err = k8s.resource(k8sconfig, delta._replace(name=""))
assert not err
# Compile the Delta and add it to the list.
create.append(DeltaCreate(delta, resource.url, local[delta]))
# Compile the Deltas to delete the excess resources.
delete = []
for meta in plan.delete:
# Resource URL. Ignore the error flag because the `strip` function
# above already called `k8s.resource` and would have aborted on error.
resource, err = k8s.resource(k8sconfig, meta)
assert not err
# Compile the Delta and add it to the list.
delete.append(DeltaDelete(meta, resource.url, del_opts.copy()))
# Iterate over each manifest that needs patching and determine the
# necessary JSON Patch to transition K8s into the state specified in the
# local manifests.
patches = []
for meta in plan.patch:
# Compute textual diff (only useful for the user to study the diff).
diff_str, err = manio.diff(config, k8sconfig, local[meta], server[meta])
if err or diff_str is None:
logit.error(f"Could not compute the diff for <{meta}>.")
return err_resp
# Compute the JSON patch that will change the K8s state to match the
# one in the local files.
patch, err = make_patch(config, k8sconfig, local[meta], server[meta])
if err or patch is None:
logit.error(f"Could not compute the patch for <{meta}>")
return err_resp
# Append the patch to the list of patches, unless it is empty.
if len(patch.ops):
patches.append(DeltaPatch(meta, diff_str, patch))
# Assemble and return the deployment plan.
return (DeploymentPlan(create, patches, delete), False)
def align_serviceaccount(
local_manifests: ServerManifests,
server_manifests: ServerManifests) -> Tuple[ServerManifests, bool]:
"""Insert the token secret from `server_manifest` into `local_manifest`.
Every ServiceAccount (SA) has a "secrets" section that K8s automatically
populates when it creates the SA. The name contains a random hash, eg
"default-token-somerandomhash" for the default service account in every
namespace.
This makes it difficult to manage service accounts with Square because the
token is not known in advance. One would have to
square apply; square plan; square get serviceaccount
to sync this, and even that is not portable because the token will be
different on a new cluster.
To avoid this problem, this function will read the token secret that K8s
added (contained in `server_manifest`) and insert it into the
`local_manifest`. This will ensure that Square creates a plan that
leaves the token secret alone.
Inputs:
local_manifests: manifests from local files that the plan will use.
server_manifests: manifests from K8s
Returns:
Copy of `local_manifests` where all ServiceAccount token secrets match
those of the server.
"""
ReturnType = Tuple[Optional[str], List[Dict[str, str]], bool]
def _get_token(meta: MetaManifest, manifests: ServerManifests) -> ReturnType:
"""Return token secret from `manifest` as well as all other other secrets.
Example input manifest:
{
'apiVersion': v1,
'kind': ServiceAccount,
...
'secrets': [
{'name': 'some-secret'},
{'name': 'demoapp-token-abcde'},
{'name': 'other-secret'},
]
}
The output for this would be:
(
'demoapp-token-abcde',
[{'name': 'some-secret'}, {'name': 'other-secret'}],
False,
)
"""
# Do nothing if the ServiceAccount has no "secrets" - should be impossible.
try:
secrets_dict = manifests[meta]["secrets"]
except KeyError:
return (None, [], False)
# Find the ServiceAccount token name.
token_prefix = f"{meta.name}-token-"
secrets = [_["name"] for _ in secrets_dict]
token = [_ for _ in secrets if _.startswith(token_prefix)]
if len(token) == 0:
# No token - return the original secrets.
return (None, secrets_dict, False)
elif len(token) == 1:
# Expected case: return the token as well as the remaining secrets.
secrets = [{"name": _} for _ in secrets if _ != token[0]]
return (token[0], secrets, False)
else:
# Unexpected.
all_secrets = str.join(", ", list(sorted(token)))
logit.warning(
f"ServiceAccount <{meta.namespace}/{meta.name}>: "
f"found multiple token secrets in: `{all_secrets}`"
)
return (None, [], True)
# Avoid side effects.
local_manifests = copy.deepcopy(local_manifests)
# Find all ServiceAccount manifests that exist locally and on the cluster.
local_meta = {k for k in local_manifests if k.kind == "ServiceAccount"}
server_meta = set(server_manifests.keys()).intersection(local_meta)
# Iterate over all ServiceAccount manifests and insert the secret token
# from the cluster into the local manifest.
for meta in server_meta:
# Find the service account token in the local/cluster manifest.
loc_token, loc_secrets, err1 = _get_token(meta, local_manifests)
srv_token, srv_secrets, err2 = _get_token(meta, server_manifests)
# Ignore the manifest if there was an error. Typically this means the
# local or cluster manifest defined multiple service account secrets.
# If that happens then something is probably seriously wrong with the
# cluster.
if err1 or err2:
continue
# Server has no token - something is probably wrong with your cluster.
if srv_token is None:
logit.warning(
f"ServiceAccount {meta.namespace}/{meta.name} has no token secret"
)
continue
# This is the expected case: the local manifest does not specify
# the token but on the cluster it exists. In that case, add the
# token here.
if srv_token and not loc_token:
loc_secrets.append({"name": srv_token})
local_manifests[meta]["secrets"] = loc_secrets
return (local_manifests, False)
def sync(local_manifests: LocalManifestLists,
server_manifests: ServerManifests,
selectors: Selectors,
groupby: GroupBy) -> Tuple[LocalManifestLists, bool]:
"""Update the local manifests with the server values and return the result.
Inputs:
local_manifests: Dict[Filepath, Tuple[MetaManifest, dict]]
server_manifests: Dict[MetaManifest, dict]
selectors: Selectors
Only operate on resources that match the selectors.
groupby: GroupBy
Specify relationship between new manifests and file names.
Returns:
Dict[Filepath, Tuple[MetaManifest, dict]]
"""
# Avoid side effects.
server_manifests = copy.deepcopy(server_manifests)
# Only retain server manifests with correct `kinds` and `namespaces`.
server_manifests = {
meta: manifest for meta, manifest in server_manifests.items()
if select(manifest, selectors)
}
# Add all local manifests outside the specified `kinds` and `namespaces`
# to the server list. This will *not* propagate to the server in any way,
# but allows us to make the rest of the function oblivious to the fact that
# we only care about a subset of namespaces and resources by pretending
# that local and server manifests are already in sync.
for fname, manifests in local_manifests.items():
for meta, manifest in manifests:
if select(manifest, selectors):
continue
server_manifests[meta] = manifest
# Create map for MetaManifest -> (File, doc-idx). The doc-idx denotes the
# index of the manifest inside the YAML files (it may contain multiple
# manifests). We will need that information later to find out which
# manifest in which file we need to update.
meta_to_fname = {}
for fname in local_manifests:
for idx, (meta, _) in enumerate(local_manifests[fname]):
meta_to_fname[meta] = (fname, idx)
del meta
del fname
# Make a copy of the local manifests to avoid side effects for the caller.
# Also put it into a default dict for convenience.
out_add_mod: DefaultDict[Filepath, List[Tuple[MetaManifest, dict]]]
out_add_mod = collections.defaultdict(list)
out_add_mod.update(copy.deepcopy(local_manifests)) # type: ignore
del local_manifests
# If the server's meta manifest exists locally then update the local one,
# otherwise add it to the catchall YAML file.
for meta, manifest in server_manifests.items():
try:
# Find the file that defined `meta` and its position inside that file.
fname, idx = meta_to_fname[meta]
except KeyError:
fname, err = filename_for_manifest(meta, manifest, groupby)
if err:
return ({}, True)
out_add_mod[fname].append((meta, manifest))
else:
# Update the correct YAML document in the correct file.
out_add_mod[fname][idx] = (meta, manifest)
# Iterate over all manifests in all files and drop the resources that do
# not exist on the server. This will, in effect, delete those resources in
# the local files if the caller chose to save them.
out_add_mod_del: LocalManifestLists = {}
for fname, manifests in out_add_mod.items():
pruned = [(meta, man) for (meta, man) in manifests if meta in server_manifests]
out_add_mod_del[fname] = pruned
return (out_add_mod_del, False)