def check_es_index():
"""
Verify that the Case and soon to be added XForm Elastic indices are up to date with what's in couch
This code is also called in the HQ admin page as well
"""
es_status = {}
es_status.update(check_es_cluster_health())
es_status.update(check_case_es_index())
es_status.update(check_xform_es_index())
es_status.update(check_reportcase_es_index())
es_status.update(check_reportxform_es_index())
do_notify = False
message = []
if es_status[CLUSTER_HEALTH] == 'red':
do_notify = True
message.append("Cluster health is red - something is up with the ES machine")
for index in es_status.keys():
if index == CLUSTER_HEALTH:
continue
pillow_status = es_status[index]
if not pillow_status['status']:
do_notify = True
message.append(
"Elasticsearch %s Index Issue: %s" % (index, es_status[index]['message']))
if do_notify:
message.append("This alert can give false alarms due to timing lag, so please double check https://www.commcarehq.org/hq/admin/system/ and the Elasticsarch Status section to make sure.")
notify_exception(None, message='\n'.join(message))
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get('api', None)
task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 12)
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
db = XFormInstance.get_db()
if type == "_active_tasks":
tasks = [] if is_bigcouch() else filter(
lambda x: x['type'] == "indexer", db.server.active_tasks())
#for reference structure is:
# tasks = [{'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 0,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023},
# {'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 70,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023}]
return json_response(tasks)
elif type == "_stats":
return json_response({})
elif type == "_logs":
pass
elif type == 'pillowtop':
return json_response(get_all_pillows_json())
elif type == 'stale_pillows':
es_index_status = [
check_case_es_index(interval=3),
check_xform_es_index(interval=3),
check_reportcase_es_index(interval=3),
check_reportxform_es_index(interval=3)
]
return json_response(es_index_status)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
if type == "flower_poll":
ret = []
try:
t = cresource.get("api/tasks",
params_dict={
'limit': task_limit
}).body_string()
all_tasks = json.loads(t)
except Exception, ex:
all_tasks = {}
logging.error("Error with getting from celery_flower: %s" % ex)
for task_id, traw in all_tasks.items():
# it's an array of arrays - looping through [<id>, {task_info_dict}]
if 'name' in traw and traw['name']:
traw['name'] = '.'.join(traw['name'].split('.')[-2:])
else:
traw['name'] = None
ret.append(traw)
ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True)
return HttpResponse(json.dumps(ret), mimetype='application/json')
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get('api', None)
task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 12)
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
db = XFormInstance.get_db()
if type == "_active_tasks":
tasks = [] if is_bigcouch() else filter(lambda x: x['type'] == "indexer", db.server.active_tasks())
#for reference structure is:
# tasks = [{'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 0,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023},
# {'type': 'indexer', 'pid': 'foo', 'database': 'mock',
# 'design_document': 'mockymock', 'progress': 70,
# 'started_on': 1349906040.723517, 'updated_on': 1349905800.679458,
# 'total_changes': 1023}]
return json_response(tasks)
elif type == "_stats":
return json_response({})
elif type == "_logs":
pass
elif type == 'pillowtop':
return json_response(get_all_pillows_json())
elif type == 'stale_pillows':
es_index_status = [
check_case_es_index(interval=3),
check_xform_es_index(interval=3),
check_reportcase_es_index(interval=3),
check_reportxform_es_index(interval=3)
]
return json_response(es_index_status)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
if type == "flower_poll":
ret = []
try:
t = cresource.get("api/tasks", params_dict={'limit': task_limit}).body_string()
all_tasks = json.loads(t)
except Exception, ex:
all_tasks = {}
logging.error("Error with getting from celery_flower: %s" % ex)
for task_id, traw in all_tasks.items():
# it's an array of arrays - looping through [<id>, {task_info_dict}]
if 'name' in traw and traw['name']:
traw['name'] = '.'.join(traw['name'].split('.')[-2:])
else:
traw['name'] = None
ret.append(traw)
ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True)
return HttpResponse(json.dumps(ret), mimetype = 'application/json')
def check_es_index():
"""
Verify that the Case and soon to be added XForm Elastic indices are up to date with what's in couch
This code is also called in the HQ admin page as well
"""
es_status = {}
es_status.update(check_es_cluster_health())
es_status.update(check_case_es_index())
es_status.update(check_xform_es_index())
es_status.update(check_reportcase_es_index())
es_status.update(check_reportxform_es_index())
do_notify = False
message = []
if es_status[CLUSTER_HEALTH] == 'red':
do_notify = True
message.append("Cluster health is red - something is up with the ES machine")
for index in es_status.keys():
if index == CLUSTER_HEALTH:
continue
pillow_status = es_status[index]
if not pillow_status['status']:
do_notify = True
message.append(
"Elasticsearch %s Index Issue: %s" % (index, es_status[index]['message']))
if do_notify:
message.append(
"This alert can give false alarms due to timing lag, so please double check "
+ absolute_reverse("system_info")
+ " and the Elasticsearch Status section to make sure."
)
notify_exception(None, message='\n'.join(message))
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get('api', None)
task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 12)
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
db = XFormInstance.get_db()
if type == "_active_tasks":
try:
tasks = [x for x in db.server.active_tasks() if x['type'] == "indexer"]
except HTTPError as e:
if e.response.status_code == 403:
return JsonResponse({'error': "Unable to access CouchDB Tasks (unauthorized)."}, status_code=500)
else:
return JsonResponse({'error': "Unable to access CouchDB Tasks."}, status_code=500)
if not is_bigcouch():
return JsonResponse(tasks, safe=False)
else:
# group tasks by design doc
task_map = defaultdict(dict)
for task in tasks:
meta = task_map[task['design_document']]
tasks = meta.get('tasks', [])
tasks.append(task)
meta['tasks'] = tasks
design_docs = []
for dd, meta in task_map.items():
meta['design_document'] = dd[len('_design/'):]
total_changes = sum(task['total_changes'] for task in meta['tasks'])
for task in meta['tasks']:
task['progress_contribution'] = task['changes_done'] * 100 // total_changes
design_docs.append(meta)
return JsonResponse(design_docs, safe=False)
elif type == "_stats":
return JsonResponse({})
elif type == "_logs":
pass
elif type == 'pillowtop':
pillow_meta = get_all_pillows_json()
return JsonResponse(sorted(pillow_meta, key=lambda m: m['name'].lower()), safe=False)
elif type == 'stale_pillows':
es_index_status = [
escheck.check_case_es_index(interval=3),
escheck.check_xform_es_index(interval=3),
escheck.check_reportcase_es_index(interval=3),
escheck.check_reportxform_es_index(interval=3)
]
return JsonResponse(es_index_status, safe=False)
if celery_monitoring:
if type == "flower_poll":
ret = []
try:
all_tasks = requests.get(
celery_monitoring + '/api/tasks',
params={'limit': task_limit},
timeout=3,
).json()
except Exception as ex:
return JsonResponse({'error': "Error with getting from celery_flower: %s" % ex}, status_code=500)
for task_id, traw in all_tasks.items():
# it's an array of arrays - looping through [<id>, {task_info_dict}]
if 'name' in traw and traw['name']:
traw['name'] = '.'.join(traw['name'].split('.')[-2:])
else:
traw['name'] = None
ret.append(traw)
ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True)
return HttpResponse(json.dumps(ret), content_type='application/json')
return HttpResponse('{}', content_type='application/json')
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get('api', None)
task_limit = getattr(settings, 'CELERYMON_TASK_LIMIT', 12)
celery_monitoring = getattr(settings, 'CELERY_FLOWER_URL', None)
db = XFormInstance.get_db()
if type == "_active_tasks":
try:
tasks = filter(lambda x: x['type'] == "indexer", db.server.active_tasks())
except Unauthorized:
return json_response({'error': "Unable to access CouchDB Tasks (unauthorized)."}, status_code=500)
if not is_bigcouch():
return json_response(tasks)
else:
# group tasks by design doc
task_map = defaultdict(dict)
for task in tasks:
meta = task_map[task['design_document']]
tasks = meta.get('tasks', [])
tasks.append(task)
meta['tasks'] = tasks
design_docs = []
for dd, meta in task_map.items():
meta['design_document'] = dd[len('_design/'):]
total_changes = sum(task['total_changes'] for task in meta['tasks'])
for task in meta['tasks']:
task['progress_contribution'] = task['changes_done'] * 100 / total_changes
design_docs.append(meta)
return json_response(design_docs)
elif type == "_stats":
return json_response({})
elif type == "_logs":
pass
elif type == 'pillowtop':
pillow_meta = get_all_pillows_json()
supervisor_status = all_pillows_supervisor_status([meta['name'] for meta in pillow_meta])
for meta in pillow_meta:
meta.update(supervisor_status[meta['name']])
return json_response(sorted(pillow_meta, key=lambda m: m['name']))
elif type == 'stale_pillows':
es_index_status = [
check_case_es_index(interval=3),
check_xform_es_index(interval=3),
check_reportcase_es_index(interval=3),
check_reportxform_es_index(interval=3)
]
return json_response(es_index_status)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
if type == "flower_poll":
ret = []
try:
t = cresource.get("api/tasks", params_dict={'limit': task_limit}).body_string()
all_tasks = json.loads(t)
except Exception, ex:
return json_response({'error': "Error with getting from celery_flower: %s" % ex}, status_code=500)
for task_id, traw in all_tasks.items():
# it's an array of arrays - looping through [<id>, {task_info_dict}]
if 'name' in traw and traw['name']:
traw['name'] = '.'.join(traw['name'].split('.')[-2:])
else:
traw['name'] = None
ret.append(traw)
ret = sorted(ret, key=lambda x: x['succeeded'], reverse=True)
return HttpResponse(json.dumps(ret), content_type='application/json')
def system_ajax(request):
"""
Utility ajax functions for polling couch and celerymon
"""
type = request.GET.get("api", None)
task_limit = getattr(settings, "CELERYMON_TASK_LIMIT", 12)
celery_monitoring = getattr(settings, "CELERY_FLOWER_URL", None)
db = XFormInstance.get_db()
if type == "_active_tasks":
try:
tasks = filter(lambda x: x["type"] == "indexer", db.server.active_tasks())
except Unauthorized:
return json_response({"error": "Unable to access CouchDB Tasks (unauthorized)."}, status_code=500)
if not is_bigcouch():
return json_response(tasks)
else:
# group tasks by design doc
task_map = defaultdict(dict)
for task in tasks:
meta = task_map[task["design_document"]]
tasks = meta.get("tasks", [])
tasks.append(task)
meta["tasks"] = tasks
design_docs = []
for dd, meta in task_map.items():
meta["design_document"] = dd[len("_design/") :]
total_changes = sum(task["total_changes"] for task in meta["tasks"])
for task in meta["tasks"]:
task["progress_contribution"] = task["changes_done"] * 100 / total_changes
design_docs.append(meta)
return json_response(design_docs)
elif type == "_stats":
return json_response({})
elif type == "_logs":
pass
elif type == "pillowtop":
return json_response(get_all_pillows_json())
elif type == "stale_pillows":
es_index_status = [
check_case_es_index(interval=3),
check_xform_es_index(interval=3),
check_reportcase_es_index(interval=3),
check_reportxform_es_index(interval=3),
]
return json_response(es_index_status)
if celery_monitoring:
cresource = Resource(celery_monitoring, timeout=3)
if type == "flower_poll":
ret = []
try:
t = cresource.get("api/tasks", params_dict={"limit": task_limit}).body_string()
all_tasks = json.loads(t)
except Exception, ex:
return json_response({"error": "Error with getting from celery_flower: %s" % ex}, status_code=500)
for task_id, traw in all_tasks.items():
# it's an array of arrays - looping through [<id>, {task_info_dict}]
if "name" in traw and traw["name"]:
traw["name"] = ".".join(traw["name"].split(".")[-2:])
else:
traw["name"] = None
ret.append(traw)
ret = sorted(ret, key=lambda x: x["succeeded"], reverse=True)
return HttpResponse(json.dumps(ret), mimetype="application/json")
