def _delete_blueprint(self, labels): """Delete a blueprint. Deletes a blueprint by deleting all CNRM resources a set of labels. We use labels because all objects belonging to a blueprint should have the same label but the names might vary. """ kinds = [ "containercluster", "iampolicymember", "iamserviceaccount", "containernodepool", "computeaddress", "computedisk" ] api_client = self.management_api for kind in kinds: client = cnrm_clients.CnrmClientApi(api_client, kind) selector = labels_to_selector(labels) logging.warning(f"Using hardcoded namespace {NAMESPACE}", extra=self._log_context) results = client.list_namespaced(NAMESPACE) if not results.items: logging.info( f"No resources of kind {kind} found to delete for " f"selector {selector}", extra=self._log_context) for i in results["items"]: name = i["metadata"]["name"] logging.info(f"Deleting kind: {kind} {NAMESPACE}.{name}", extra=self._log_context) client.delete_namespaced(NAMESPACE, name, {})
def _iter_blueprints(namespace, context=None): """Return an iterator over blueprints. Args: namespace: The namespace to look for blueprints context: The kube context to use. """ # We need to load the kube config so that we can have credentials to # talk to the APIServer. util.load_kube_config(persist_config=False, context=context) client = k8s_client.ApiClient() crd_api = cnrm_clients.CnrmClientApi(client, "containercluster") clusters = crd_api.list_namespaced(namespace) for c in clusters.get("items"): yield c
def _iter_blueprints(self): """Return an iterator over blueprints. Args: namespace: The namespace to look for blueprints context: The kube context to use. """ # We need to load the kube config so that we can have credentials to # talk to the APIServer. crd_api = cnrm_clients.CnrmClientApi(self.management_api, "containercluster") # Treat all clusters with an auto-group-label as an auto deployed group. selector = GROUP_LABEL namespace = "kubeflow-ci-deployment" # TODO(jlewi): We should get namespaces by looking at the PipelineRuns # to get the project it is deployed to. logging.warning(f"Using hardcoded value of namespace={namespace} to look " f"for deployments.") clusters = crd_api.list_namespaced(namespace, label_selector=selector) for c in clusters.get("items"): yield c
def _delete_blueprints(namespace, to_keep_names, context=None, dryrun=True): """Delete all auto-deployed resources that we don't want to keep. Args: namespace: The namespace that owns the CNRM objects. to_keep_names: Names of the blueprints to keep. context: The kubeconfig context to use This function deletes all auto-deployed resources that we don't want to keep. This function is intended to delete any orphaned resources. It works as follows. 1. For each type of resource we issue a list to find all autodeployed resources 2. We then remove any resource which belongs to a blueprint to keep 3. We remove any resource that is less than 1 hours old * This is to avoid race conditions where a blueprint was created after to_keep was computedisks 4. remaining resources are deleted. """ util.load_kube_config(persist_config=False, context=context) client = k8s_client.ApiClient() crd_api = k8s_client.CustomObjectsApi(client) BASE_GROUP = "cnrm.cloud.google.com" CNRM_VERSION = "v1beta1" # List of resources to GC kinds = [ "containercluster", "iampolicymember", "iamserviceaccount", "containernodepool", "computeaddress", "computedisk" ] # Mappings from resource type to list of resources to_keep = collections.defaultdict(lambda: []) to_delete = collections.defaultdict(lambda: []) api_client = k8s_client.ApiClient() # Loop over resources and identify resources to delete. for kind in kinds: client = cnrm_clients.CnrmClientApi(api_client, kind) selector = "{0}=true".format(AUTO_DEPLOY_LABEL) results = client.list_namespaced(namespace, label_selector=selector) for i in results.get("items"): name = i["metadata"]["name"] if name in to_keep_names: to_keep[kind].append(name) continue creation = date_parser.parse(i["metadata"]["creationTimestamp"]) age = datetime.datetime.now(creation.tzinfo) - creation if age < datetime.timedelta(hours=1): to_keep[kind].append(name) logging.info("Not GC'ing %s %s; it was created to recently", kind, name) continue to_delete[kind].append(name) for kind in kinds: client = cnrm_clients.CnrmClientApi(api_client, kind) for name in to_delete[kind]: if dryrun: logging.info("Dryrun: %s %s would be deleted", kind, name) else: logging.info("Deleting: %s %s", kind, name) client.delete_namespaced(namespace, name, {}) for kind in kinds: logging.info("Deleted %s:\n%s", kind, "\n".join(to_delete[kind])) logging.info("Kept %s:\n%s", kind, "\n".join(to_keep[kind]))