def page_context(self):
environment = settings.SERVER_ENVIRONMENT
context = get_hqadmin_base_context(self.request)
context['couch_update'] = self.request.GET.get('couch_update', 5000)
context['celery_update'] = self.request.GET.get('celery_update', 10000)
context['db_update'] = self.request.GET.get('db_update', 30000)
context['self.request'] = getattr(settings, 'CELERY_FLOWER_URL', None)
context['is_bigcouch'] = is_bigcouch()
context['rabbitmq_url'] = get_rabbitmq_management_url()
context['hide_filters'] = True
context['current_system'] = socket.gethostname()
context['deploy_history'] = HqDeploy.objects.filter(environment=environment)[:5]
context['user_is_support'] = hasattr(self.request, 'user') and SUPPORT.enabled(self.request.user.username)
context['redis'] = service_checks.check_redis()
context['rabbitmq'] = service_checks.check_rabbitmq(settings.CELERY_BROKER_URL)
context['celery_stats'] = get_celery_stats()
context['heartbeat'] = service_checks.check_heartbeat()
context['cluster_health'] = escheck.check_es_cluster_health()
return context
def check_es_index():
"""
Verify that the Case and soon to be added XForm Elastic indices are up to date with what's in couch
This code is also called in the HQ admin page as well
"""
es_status = {}
es_status.update(check_es_cluster_health())
es_status.update(check_case_es_index())
es_status.update(check_xform_es_index())
es_status.update(check_reportcase_es_index())
es_status.update(check_reportxform_es_index())
do_notify = False
message = []
if es_status[CLUSTER_HEALTH] == 'red':
do_notify = True
message.append("Cluster health is red - something is up with the ES machine")
for index in es_status.keys():
if index == CLUSTER_HEALTH:
continue
pillow_status = es_status[index]
if not pillow_status['status']:
do_notify = True
message.append(
"Elasticsearch %s Index Issue: %s" % (index, es_status[index]['message']))
if do_notify:
message.append("This alert can give false alarms due to timing lag, so please double check https://www.commcarehq.org/hq/admin/system/ and the Elasticsarch Status section to make sure.")
notify_exception(None, message='\n'.join(message))
def system_info(request):
environment = settings.SERVER_ENVIRONMENT
context = get_hqadmin_base_context(request)
context['couch_update'] = request.GET.get('couch_update', 5000)
context['celery_update'] = request.GET.get('celery_update', 10000)
context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None)
# recent changes
recent_changes = int(request.GET.get('changes', 50))
context['recent_changes'] = get_recent_changes(get_db(), recent_changes)
context['rabbitmq_url'] = get_rabbitmq_management_url()
context['hide_filters'] = True
context['current_system'] = socket.gethostname()
context['last_deploy'] = HqDeploy.get_latest(environment)
context['snapshot'] = context['last_deploy'].code_snapshot if context[
'last_deploy'] else {}
context.update(check_redis())
context.update(check_rabbitmq())
context.update(check_celery_health())
context.update(check_memcached())
context.update(check_es_cluster_health())
return render(request, "hqadmin/system_info.html", context)
def page_context(self):
environment = settings.SERVER_ENVIRONMENT
context = get_hqadmin_base_context(self.request)
context['couch_update'] = self.request.GET.get('couch_update', 5000)
context['celery_update'] = self.request.GET.get('celery_update', 10000)
context['db_update'] = self.request.GET.get('db_update', 30000)
context['self.request'] = getattr(settings, 'CELERY_FLOWER_URL', None)
context['is_bigcouch'] = is_bigcouch()
context['rabbitmq_url'] = get_rabbitmq_management_url()
context['hide_filters'] = True
context['current_system'] = socket.gethostname()
context['deploy_history'] = HqDeploy.get_latest(environment, limit=5)
context['user_is_support'] = hasattr(self.request, 'user') and SUPPORT.enabled(self.request.user.username)
context['redis'] = service_checks.check_redis()
context['rabbitmq'] = service_checks.check_rabbitmq()
context['celery_stats'] = get_celery_stats()
context['heartbeat'] = service_checks.check_heartbeat()
context['cluster_health'] = escheck.check_es_cluster_health()
return context
def check_elasticsearch():
cluster_health = check_es_cluster_health()
if cluster_health != 'green':
return ServiceStatus(False, "Cluster health at %s" % cluster_health)
doc = {'_id': 'elasticsearch-service-check',
'date': datetime.datetime.now().isoformat()}
send_to_elasticsearch('groups', doc)
refresh_elasticsearch_index('groups')
hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits
send_to_elasticsearch('groups', doc, delete=True) # clean up
if doc in hits:
return ServiceStatus(True, "Successfully sent a doc to ES and read it back")
return ServiceStatus(False, "Something went wrong sending a doc to ES")
def check_elasticsearch():
cluster_health = check_es_cluster_health()
if cluster_health != 'green':
return ServiceStatus(False, "Cluster health at %s" % cluster_health)
doc = {'_id': 'elasticsearch-service-check-{}'.format(random_hex()[:7]),
'date': datetime.datetime.now().isoformat()}
try:
send_to_elasticsearch('groups', doc)
refresh_elasticsearch_index('groups')
hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits
if doc in hits:
return ServiceStatus(True, "Successfully sent a doc to ES and read it back")
else:
return ServiceStatus(False, "Something went wrong sending a doc to ES")
finally:
send_to_elasticsearch('groups', doc, delete=True) # clean up
def system_info(request):
environment = settings.SERVER_ENVIRONMENT
context = get_hqadmin_base_context(request)
context['couch_update'] = request.GET.get('couch_update', 5000)
context['celery_update'] = request.GET.get('celery_update', 10000)
context['db_update'] = request.GET.get('db_update', 30000)
context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None)
context['is_bigcouch'] = is_bigcouch()
context['rabbitmq_url'] = get_rabbitmq_management_url()
context['hide_filters'] = True
context['current_system'] = socket.gethostname()
context['deploy_history'] = HqDeploy.get_latest(environment, limit=5)
context.update(check_redis())
context.update(check_rabbitmq())
context.update(check_celery_health())
context.update(check_es_cluster_health())
return render(request, "hqadmin/system_info.html", context)
def check_es_index():
"""
Verify that the Case and soon to be added XForm Elastic indices are up to date with what's in couch
This code is also called in the HQ admin page as well
"""
es_status = {}
es_status.update(check_es_cluster_health())
es_status.update(check_case_es_index())
es_status.update(check_xform_es_index())
es_status.update(check_reportcase_es_index())
es_status.update(check_reportxform_es_index())
do_notify = False
message = []
if es_status[CLUSTER_HEALTH] == 'red':
do_notify = True
message.append("Cluster health is red - something is up with the ES machine")
for index in es_status.keys():
if index == CLUSTER_HEALTH:
continue
pillow_status = es_status[index]
if not pillow_status['status']:
do_notify = True
message.append(
"Elasticsearch %s Index Issue: %s" % (index, es_status[index]['message']))
if do_notify:
message.append(
"This alert can give false alarms due to timing lag, so please double check "
+ absolute_reverse("system_info")
+ " and the Elasticsearch Status section to make sure."
)
notify_exception(None, message='\n'.join(message))
def system_info(request):
environment = settings.SERVER_ENVIRONMENT
context = get_hqadmin_base_context(request)
context['couch_update'] = request.GET.get('couch_update', 5000)
context['celery_update'] = request.GET.get('celery_update', 10000)
context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None)
# recent changes
recent_changes = int(request.GET.get('changes', 50))
context['recent_changes'] = get_recent_changes(get_db(), recent_changes)
context['rabbitmq_url'] = get_rabbitmq_management_url()
context['hide_filters'] = True
context['current_system'] = socket.gethostname()
context['last_deploy'] = HqDeploy.get_latest(environment)
context['snapshot'] = context['last_deploy'].code_snapshot if context['last_deploy'] else {}
context.update(check_redis())
context.update(check_rabbitmq())
context.update(check_celery_health())
context.update(check_memcached())
context.update(check_es_cluster_health())
return render(request, "hqadmin/system_info.html", context)
