def get(instance_id, instance, res): res['ready'] = True app_pods_status = {} for pod in kubectl.get('pods', namespace=instance_id, required=True)['items']: app = pod['metadata']['labels'].get('app') if not app: app = 'unknown' item_status = kubectl.get_item_detailed_status(pod) if item_status.get('errors') and len(item_status['errors']) > 0: res['ready'] = False app_pods_status.setdefault(app, {})[pod['metadata']['name']] = item_status app_deployments_status = {} for deployment in kubectl.get('deployments', namespace=instance_id, required=True)['items']: app = deployment['metadata']['labels'].get('app') if not app: app = 'unknown' item_status = kubectl.get_item_detailed_status(deployment) if item_status.get('errors') and len(item_status['errors']) > 0: res['ready'] = False app_deployments_status.setdefault( app, {})[deployment['metadata']['name']] = item_status if 'jenkins' not in app_pods_status: res['ready'] = False res['app'] = { 'pods': app_pods_status, 'deployments': app_deployments_status }
def get(self): exitcode, output = subprocess.getstatusoutput( f'kubectl -n {self.instance.id} get deployment/{self.instance.id} -o yaml' ) if exitcode == 0: deployment = yaml.load(output) status = kubectl.get_item_detailed_status(deployment) ready = len(status.get('error', [])) == 0 status['pods'] = [] pods = kubectl.get('pods -l app=ckan', namespace=self.instance.id, required=False) image = None latest_operator_timestamp, latest_pod_name, latest_pod_status = None, None, None if pods: for pod in pods['items']: pod_operator_timestamp = pod['metadata']['annotations'][ 'ckan-cloud/operator-timestamp'] if not latest_operator_timestamp or latest_operator_timestamp < pod_operator_timestamp: latest_operator_timestamp = pod_operator_timestamp latest_pod_name = pod['metadata']['name'] pod_status = kubectl.get_item_detailed_status(pod) status_code, output = subprocess.getstatusoutput( f'kubectl -n {self.instance.id} logs {pod["metadata"]["name"]} -c ckan --tail 5', ) if status_code == 0: pod_status['logs'] = output else: pod_status['logs'] = None if not image: image = pod["spec"]["containers"][0]["image"] else: if image != pod["spec"]["containers"][0]["image"]: ready = False image = pod["spec"]["containers"][0]["image"] status['pods'].append(pod_status) if latest_pod_name == pod_status['name']: latest_pod_status = pod_status if not latest_pod_status or len( latest_pod_status.get( 'errors', [])) > 0 or latest_pod_status['logs'] is None: ready = False else: ready = False return dict(status, ready=ready, image=image, latest_pod_name=latest_pod_name, latest_operator_timestamp=latest_operator_timestamp) else: return {'ready': False, 'error': output}
def get(instance_id, instance, res): namespace = instance["spec"]["namespace"] storage_class = instance["spec"]["storageclass"] res['ready'] = True app_pod = None for pod in kubectl.get('pods', namespace=namespace, required=True)['items']: app = pod['metadata']['labels'].get('app') if app == instance_id: assert app_pod is None, "Too many nfs-client-provisioner pods in namespace %s" % namespace app_pod = pod assert app_pod is not None, "nfs-client-provisioner pod is missing from namespace %s" % namespace pod_status = kubectl.get_item_detailed_status(app_pod) app_deployment = kubectl.get("deployment", instance_id, namespace=namespace, required=True) deployment_status = kubectl.get_item_detailed_status(app_deployment) storage_class = kubectl.get("storageclass", storage_class, namespace=namespace, required=True) storage_class_status = { "name": storage_class["metadata"]["name"], "parameters": storage_class["parameters"], "reclaimPolicy": storage_class["reclaimPolicy"], } for status in (pod_status, deployment_status, storage_class_status): if status.get("errors") and len(status["errors"]) > 0: res["ready"] = False res['app'] = { 'namespace': namespace, 'pod': pod_status, 'deployment': deployment_status, "storage_class": storage_class_status }
def get(instance_id, instance=None): image = None latest_operator_timestamp, latest_pod_name, latest_pod_status = None, None, None item_app_statuses = {} ckan_deployment_status = None ckan_deployment_ready = None ckan_deployment_status_pods = [] logs.debug('Getting all namespace resources', namespace=instance_id) all_resources = kubectl.get('all', namespace=instance_id, required=False) num_resource_items = len(all_resources.get('items')) logs.debug(num_resource_items=num_resource_items) if num_resource_items > 0: for item in all_resources['items']: item_kind = item['kind'] try: if item_kind in ("Pod", "ReplicaSet"): item_app = item["metadata"]["labels"]["app"] elif item_kind in ("Service", "Deployment"): item_app = item["metadata"]["name"] else: item_app = None except: logging.exception('Failed to extract item_app from %r', item) item_app = None logs.debug(item_kind=item_kind, item_app=item_app) if item_app in ["ckan", "jobs-db", "redis", "nginx", "jobs"]: app_status = item_app_statuses.setdefault(item_app, {}) else: app_status = item_app_statuses.setdefault("unknown", {}) item_status = kubectl.get_item_detailed_status(item) app_status.setdefault("{}s".format(item_kind.lower()), []).append(item_status) if item_app == 'ckan': if item_kind == 'Deployment': ckan_deployment_status = item_status ckan_deployment_ready = len(item_status.get('error', [])) == 0 logs.debug(ckan_deployment_ready=ckan_deployment_ready) elif item_kind == 'Pod': pod = item pod_status = item_status pod_operator_timestamp = pod['metadata'][ 'creationTimestamp'] if not latest_operator_timestamp or latest_operator_timestamp < pod_operator_timestamp: latest_operator_timestamp = pod_operator_timestamp latest_pod_name = pod['metadata']['name'] for container in ["secrets", "ckan"]: status_code, output = subprocess.getstatusoutput( f'kubectl -n {instance_id} logs {pod["metadata"]["name"]} -c {container}', ) container_logs = output if status_code == 0 else None logs.debug(len_container_logs=len(container_logs) if container_logs else 0) if container == 'ckan': pod_status['logs'] = output if container_logs: for logline in container_logs.split( "--START_CKAN_CLOUD_LOG--")[1:]: logdata = json.loads( logline.split("--END_CKAN_CLOUD_LOG--")[0]) pod_status.setdefault("ckan-cloud-logs", []).append(logdata) if not image: image = pod["spec"]["containers"][0]["image"] else: if image != pod["spec"]["containers"][0]["image"]: ckan_deployment_ready = False image = pod["spec"]["containers"][0]["image"] ckan_deployment_status_pods.append(pod_status) if latest_pod_name == pod_status['name']: latest_pod_status = pod_status if not latest_pod_status or len(latest_pod_status.get( 'errors', [])) > 0 or latest_pod_status['logs'] is None: ckan_deployment_ready = False else: ckan_deployment_ready = False return { **ckan_deployment_status, 'ready': ckan_deployment_ready, 'pods': ckan_deployment_status_pods, 'image': image, 'latest_pod_name': latest_pod_name, 'latest_operator_timestamp': latest_operator_timestamp, 'helm_app_statuses': item_app_statuses, 'helm_metadata': { 'ckan_instance_id': instance_id, 'namespace': instance_id, 'status_generated_at': datetime.datetime.now(), 'status_generated_from': subprocess.check_output(["hostname"]).decode().strip(), } }