def get(instance_id, instance, res):
res['ready'] = True
app_pods_status = {}
for pod in kubectl.get('pods', namespace=instance_id,
required=True)['items']:
app = pod['metadata']['labels'].get('app')
if not app:
app = 'unknown'
item_status = kubectl.get_item_detailed_status(pod)
if item_status.get('errors') and len(item_status['errors']) > 0:
res['ready'] = False
app_pods_status.setdefault(app,
{})[pod['metadata']['name']] = item_status
app_deployments_status = {}
for deployment in kubectl.get('deployments',
namespace=instance_id,
required=True)['items']:
app = deployment['metadata']['labels'].get('app')
if not app:
app = 'unknown'
item_status = kubectl.get_item_detailed_status(deployment)
if item_status.get('errors') and len(item_status['errors']) > 0:
res['ready'] = False
app_deployments_status.setdefault(
app, {})[deployment['metadata']['name']] = item_status
if 'jenkins' not in app_pods_status:
res['ready'] = False
res['app'] = {
'pods': app_pods_status,
'deployments': app_deployments_status
}
def get(self):
exitcode, output = subprocess.getstatusoutput(
f'kubectl -n {self.instance.id} get deployment/{self.instance.id} -o yaml'
)
if exitcode == 0:
deployment = yaml.load(output)
status = kubectl.get_item_detailed_status(deployment)
ready = len(status.get('error', [])) == 0
status['pods'] = []
pods = kubectl.get('pods -l app=ckan',
namespace=self.instance.id,
required=False)
image = None
latest_operator_timestamp, latest_pod_name, latest_pod_status = None, None, None
if pods:
for pod in pods['items']:
pod_operator_timestamp = pod['metadata']['annotations'][
'ckan-cloud/operator-timestamp']
if not latest_operator_timestamp or latest_operator_timestamp < pod_operator_timestamp:
latest_operator_timestamp = pod_operator_timestamp
latest_pod_name = pod['metadata']['name']
pod_status = kubectl.get_item_detailed_status(pod)
status_code, output = subprocess.getstatusoutput(
f'kubectl -n {self.instance.id} logs {pod["metadata"]["name"]} -c ckan --tail 5',
)
if status_code == 0:
pod_status['logs'] = output
else:
pod_status['logs'] = None
if not image:
image = pod["spec"]["containers"][0]["image"]
else:
if image != pod["spec"]["containers"][0]["image"]:
ready = False
image = pod["spec"]["containers"][0]["image"]
status['pods'].append(pod_status)
if latest_pod_name == pod_status['name']:
latest_pod_status = pod_status
if not latest_pod_status or len(
latest_pod_status.get(
'errors',
[])) > 0 or latest_pod_status['logs'] is None:
ready = False
else:
ready = False
return dict(status,
ready=ready,
image=image,
latest_pod_name=latest_pod_name,
latest_operator_timestamp=latest_operator_timestamp)
else:
return {'ready': False, 'error': output}
def get(instance_id, instance, res):
namespace = instance["spec"]["namespace"]
storage_class = instance["spec"]["storageclass"]
res['ready'] = True
app_pod = None
for pod in kubectl.get('pods', namespace=namespace,
required=True)['items']:
app = pod['metadata']['labels'].get('app')
if app == instance_id:
assert app_pod is None, "Too many nfs-client-provisioner pods in namespace %s" % namespace
app_pod = pod
assert app_pod is not None, "nfs-client-provisioner pod is missing from namespace %s" % namespace
pod_status = kubectl.get_item_detailed_status(app_pod)
app_deployment = kubectl.get("deployment",
instance_id,
namespace=namespace,
required=True)
deployment_status = kubectl.get_item_detailed_status(app_deployment)
storage_class = kubectl.get("storageclass",
storage_class,
namespace=namespace,
required=True)
storage_class_status = {
"name": storage_class["metadata"]["name"],
"parameters": storage_class["parameters"],
"reclaimPolicy": storage_class["reclaimPolicy"],
}
for status in (pod_status, deployment_status, storage_class_status):
if status.get("errors") and len(status["errors"]) > 0:
res["ready"] = False
res['app'] = {
'namespace': namespace,
'pod': pod_status,
'deployment': deployment_status,
"storage_class": storage_class_status
}
def get(instance_id, instance=None):
image = None
latest_operator_timestamp, latest_pod_name, latest_pod_status = None, None, None
item_app_statuses = {}
ckan_deployment_status = None
ckan_deployment_ready = None
ckan_deployment_status_pods = []
logs.debug('Getting all namespace resources', namespace=instance_id)
all_resources = kubectl.get('all', namespace=instance_id, required=False)
num_resource_items = len(all_resources.get('items'))
logs.debug(num_resource_items=num_resource_items)
if num_resource_items > 0:
for item in all_resources['items']:
item_kind = item['kind']
try:
if item_kind in ("Pod", "ReplicaSet"):
item_app = item["metadata"]["labels"]["app"]
elif item_kind in ("Service", "Deployment"):
item_app = item["metadata"]["name"]
else:
item_app = None
except:
logging.exception('Failed to extract item_app from %r', item)
item_app = None
logs.debug(item_kind=item_kind, item_app=item_app)
if item_app in ["ckan", "jobs-db", "redis", "nginx", "jobs"]:
app_status = item_app_statuses.setdefault(item_app, {})
else:
app_status = item_app_statuses.setdefault("unknown", {})
item_status = kubectl.get_item_detailed_status(item)
app_status.setdefault("{}s".format(item_kind.lower()),
[]).append(item_status)
if item_app == 'ckan':
if item_kind == 'Deployment':
ckan_deployment_status = item_status
ckan_deployment_ready = len(item_status.get('error',
[])) == 0
logs.debug(ckan_deployment_ready=ckan_deployment_ready)
elif item_kind == 'Pod':
pod = item
pod_status = item_status
pod_operator_timestamp = pod['metadata'][
'creationTimestamp']
if not latest_operator_timestamp or latest_operator_timestamp < pod_operator_timestamp:
latest_operator_timestamp = pod_operator_timestamp
latest_pod_name = pod['metadata']['name']
for container in ["secrets", "ckan"]:
status_code, output = subprocess.getstatusoutput(
f'kubectl -n {instance_id} logs {pod["metadata"]["name"]} -c {container}',
)
container_logs = output if status_code == 0 else None
logs.debug(len_container_logs=len(container_logs)
if container_logs else 0)
if container == 'ckan':
pod_status['logs'] = output
if container_logs:
for logline in container_logs.split(
"--START_CKAN_CLOUD_LOG--")[1:]:
logdata = json.loads(
logline.split("--END_CKAN_CLOUD_LOG--")[0])
pod_status.setdefault("ckan-cloud-logs",
[]).append(logdata)
if not image:
image = pod["spec"]["containers"][0]["image"]
else:
if image != pod["spec"]["containers"][0]["image"]:
ckan_deployment_ready = False
image = pod["spec"]["containers"][0]["image"]
ckan_deployment_status_pods.append(pod_status)
if latest_pod_name == pod_status['name']:
latest_pod_status = pod_status
if not latest_pod_status or len(latest_pod_status.get(
'errors', [])) > 0 or latest_pod_status['logs'] is None:
ckan_deployment_ready = False
else:
ckan_deployment_ready = False
return {
**ckan_deployment_status, 'ready': ckan_deployment_ready,
'pods': ckan_deployment_status_pods,
'image': image,
'latest_pod_name': latest_pod_name,
'latest_operator_timestamp': latest_operator_timestamp,
'helm_app_statuses': item_app_statuses,
'helm_metadata': {
'ckan_instance_id':
instance_id,
'namespace':
instance_id,
'status_generated_at':
datetime.datetime.now(),
'status_generated_from':
subprocess.check_output(["hostname"]).decode().strip(),
}
}