def _delete_blueprint(self, labels):
"""Delete a blueprint.
Deletes a blueprint by deleting all CNRM resources a set of labels.
We use labels because all objects belonging to a blueprint should have
the same label but the names might vary.
"""
kinds = [
"containercluster", "iampolicymember", "iamserviceaccount",
"containernodepool", "computeaddress", "computedisk"
]
api_client = self.management_api
for kind in kinds:
client = cnrm_clients.CnrmClientApi(api_client, kind)
selector = labels_to_selector(labels)
logging.warning(f"Using hardcoded namespace {NAMESPACE}",
extra=self._log_context)
results = client.list_namespaced(NAMESPACE)
if not results.items:
logging.info(
f"No resources of kind {kind} found to delete for "
f"selector {selector}",
extra=self._log_context)
for i in results["items"]:
name = i["metadata"]["name"]
logging.info(f"Deleting kind: {kind} {NAMESPACE}.{name}",
extra=self._log_context)
client.delete_namespaced(NAMESPACE, name, {})
def _iter_blueprints(namespace, context=None):
"""Return an iterator over blueprints.
Args:
namespace: The namespace to look for blueprints
context: The kube context to use.
"""
# We need to load the kube config so that we can have credentials to
# talk to the APIServer.
util.load_kube_config(persist_config=False, context=context)
client = k8s_client.ApiClient()
crd_api = cnrm_clients.CnrmClientApi(client, "containercluster")
clusters = crd_api.list_namespaced(namespace)
for c in clusters.get("items"):
yield c
def _iter_blueprints(self):
"""Return an iterator over blueprints.
Args:
namespace: The namespace to look for blueprints
context: The kube context to use.
"""
# We need to load the kube config so that we can have credentials to
# talk to the APIServer.
crd_api = cnrm_clients.CnrmClientApi(self.management_api, "containercluster")
# Treat all clusters with an auto-group-label as an auto deployed group.
selector = GROUP_LABEL
namespace = "kubeflow-ci-deployment"
# TODO(jlewi): We should get namespaces by looking at the PipelineRuns
# to get the project it is deployed to.
logging.warning(f"Using hardcoded value of namespace={namespace} to look "
f"for deployments.")
clusters = crd_api.list_namespaced(namespace, label_selector=selector)
for c in clusters.get("items"):
yield c
def _delete_blueprints(namespace, to_keep_names, context=None, dryrun=True):
"""Delete all auto-deployed resources that we don't want to keep.
Args:
namespace: The namespace that owns the CNRM objects.
to_keep_names: Names of the blueprints to keep.
context: The kubeconfig context to use
This function deletes all auto-deployed resources that we don't want
to keep. This function is intended to delete any orphaned resources.
It works as follows.
1. For each type of resource we issue a list to find all autodeployed
resources
2. We then remove any resource which belongs to a blueprint to keep
3. We remove any resource that is less than 1 hours old
* This is to avoid race conditions where a blueprint was created
after to_keep was computedisks
4. remaining resources are deleted.
"""
util.load_kube_config(persist_config=False, context=context)
client = k8s_client.ApiClient()
crd_api = k8s_client.CustomObjectsApi(client)
BASE_GROUP = "cnrm.cloud.google.com"
CNRM_VERSION = "v1beta1"
# List of resources to GC
kinds = [
"containercluster", "iampolicymember", "iamserviceaccount",
"containernodepool", "computeaddress", "computedisk"
]
# Mappings from resource type to list of resources
to_keep = collections.defaultdict(lambda: [])
to_delete = collections.defaultdict(lambda: [])
api_client = k8s_client.ApiClient()
# Loop over resources and identify resources to delete.
for kind in kinds:
client = cnrm_clients.CnrmClientApi(api_client, kind)
selector = "{0}=true".format(AUTO_DEPLOY_LABEL)
results = client.list_namespaced(namespace, label_selector=selector)
for i in results.get("items"):
name = i["metadata"]["name"]
if name in to_keep_names:
to_keep[kind].append(name)
continue
creation = date_parser.parse(i["metadata"]["creationTimestamp"])
age = datetime.datetime.now(creation.tzinfo) - creation
if age < datetime.timedelta(hours=1):
to_keep[kind].append(name)
logging.info("Not GC'ing %s %s; it was created to recently",
kind, name)
continue
to_delete[kind].append(name)
for kind in kinds:
client = cnrm_clients.CnrmClientApi(api_client, kind)
for name in to_delete[kind]:
if dryrun:
logging.info("Dryrun: %s %s would be deleted", kind, name)
else:
logging.info("Deleting: %s %s", kind, name)
client.delete_namespaced(namespace, name, {})
for kind in kinds:
logging.info("Deleted %s:\n%s", kind, "\n".join(to_delete[kind]))
logging.info("Kept %s:\n%s", kind, "\n".join(to_keep[kind]))