def page_context(self): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(self.request) context['couch_update'] = self.request.GET.get('couch_update', 5000) context['celery_update'] = self.request.GET.get('celery_update', 10000) context['db_update'] = self.request.GET.get('db_update', 30000) context['self.request'] = getattr(settings, 'CELERY_FLOWER_URL', None) context['is_bigcouch'] = is_bigcouch() context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['deploy_history'] = HqDeploy.objects.filter(environment=environment)[:5] context['user_is_support'] = hasattr(self.request, 'user') and SUPPORT.enabled(self.request.user.username) context['redis'] = service_checks.check_redis() context['rabbitmq'] = service_checks.check_rabbitmq(settings.CELERY_BROKER_URL) context['celery_stats'] = get_celery_stats() context['heartbeat'] = service_checks.check_heartbeat() context['cluster_health'] = escheck.check_es_cluster_health() return context
def check_es_index(): """ Verify that the Case and soon to be added XForm Elastic indices are up to date with what's in couch This code is also called in the HQ admin page as well """ es_status = {} es_status.update(check_es_cluster_health()) es_status.update(check_case_es_index()) es_status.update(check_xform_es_index()) es_status.update(check_reportcase_es_index()) es_status.update(check_reportxform_es_index()) do_notify = False message = [] if es_status[CLUSTER_HEALTH] == 'red': do_notify = True message.append("Cluster health is red - something is up with the ES machine") for index in es_status.keys(): if index == CLUSTER_HEALTH: continue pillow_status = es_status[index] if not pillow_status['status']: do_notify = True message.append( "Elasticsearch %s Index Issue: %s" % (index, es_status[index]['message'])) if do_notify: message.append("This alert can give false alarms due to timing lag, so please double check https://www.commcarehq.org/hq/admin/system/ and the Elasticsarch Status section to make sure.") notify_exception(None, message='\n'.join(message))
def system_info(request): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(request) context['couch_update'] = request.GET.get('couch_update', 5000) context['celery_update'] = request.GET.get('celery_update', 10000) context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None) # recent changes recent_changes = int(request.GET.get('changes', 50)) context['recent_changes'] = get_recent_changes(get_db(), recent_changes) context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['last_deploy'] = HqDeploy.get_latest(environment) context['snapshot'] = context['last_deploy'].code_snapshot if context[ 'last_deploy'] else {} context.update(check_redis()) context.update(check_rabbitmq()) context.update(check_celery_health()) context.update(check_memcached()) context.update(check_es_cluster_health()) return render(request, "hqadmin/system_info.html", context)
def page_context(self): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(self.request) context['couch_update'] = self.request.GET.get('couch_update', 5000) context['celery_update'] = self.request.GET.get('celery_update', 10000) context['db_update'] = self.request.GET.get('db_update', 30000) context['self.request'] = getattr(settings, 'CELERY_FLOWER_URL', None) context['is_bigcouch'] = is_bigcouch() context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['deploy_history'] = HqDeploy.get_latest(environment, limit=5) context['user_is_support'] = hasattr(self.request, 'user') and SUPPORT.enabled(self.request.user.username) context['redis'] = service_checks.check_redis() context['rabbitmq'] = service_checks.check_rabbitmq() context['celery_stats'] = get_celery_stats() context['heartbeat'] = service_checks.check_heartbeat() context['cluster_health'] = escheck.check_es_cluster_health() return context
def check_elasticsearch(): cluster_health = check_es_cluster_health() if cluster_health != 'green': return ServiceStatus(False, "Cluster health at %s" % cluster_health) doc = {'_id': 'elasticsearch-service-check', 'date': datetime.datetime.now().isoformat()} send_to_elasticsearch('groups', doc) refresh_elasticsearch_index('groups') hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits send_to_elasticsearch('groups', doc, delete=True) # clean up if doc in hits: return ServiceStatus(True, "Successfully sent a doc to ES and read it back") return ServiceStatus(False, "Something went wrong sending a doc to ES")
def check_elasticsearch(): cluster_health = check_es_cluster_health() if cluster_health != 'green': return ServiceStatus(False, "Cluster health at %s" % cluster_health) doc = {'_id': 'elasticsearch-service-check-{}'.format(random_hex()[:7]), 'date': datetime.datetime.now().isoformat()} try: send_to_elasticsearch('groups', doc) refresh_elasticsearch_index('groups') hits = GroupES().remove_default_filters().doc_id(doc['_id']).run().hits if doc in hits: return ServiceStatus(True, "Successfully sent a doc to ES and read it back") else: return ServiceStatus(False, "Something went wrong sending a doc to ES") finally: send_to_elasticsearch('groups', doc, delete=True) # clean up
def system_info(request): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(request) context['couch_update'] = request.GET.get('couch_update', 5000) context['celery_update'] = request.GET.get('celery_update', 10000) context['db_update'] = request.GET.get('db_update', 30000) context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None) context['is_bigcouch'] = is_bigcouch() context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['deploy_history'] = HqDeploy.get_latest(environment, limit=5) context.update(check_redis()) context.update(check_rabbitmq()) context.update(check_celery_health()) context.update(check_es_cluster_health()) return render(request, "hqadmin/system_info.html", context)
def check_es_index(): """ Verify that the Case and soon to be added XForm Elastic indices are up to date with what's in couch This code is also called in the HQ admin page as well """ es_status = {} es_status.update(check_es_cluster_health()) es_status.update(check_case_es_index()) es_status.update(check_xform_es_index()) es_status.update(check_reportcase_es_index()) es_status.update(check_reportxform_es_index()) do_notify = False message = [] if es_status[CLUSTER_HEALTH] == 'red': do_notify = True message.append("Cluster health is red - something is up with the ES machine") for index in es_status.keys(): if index == CLUSTER_HEALTH: continue pillow_status = es_status[index] if not pillow_status['status']: do_notify = True message.append( "Elasticsearch %s Index Issue: %s" % (index, es_status[index]['message'])) if do_notify: message.append( "This alert can give false alarms due to timing lag, so please double check " + absolute_reverse("system_info") + " and the Elasticsearch Status section to make sure." ) notify_exception(None, message='\n'.join(message))
def system_info(request): environment = settings.SERVER_ENVIRONMENT context = get_hqadmin_base_context(request) context['couch_update'] = request.GET.get('couch_update', 5000) context['celery_update'] = request.GET.get('celery_update', 10000) context['celery_flower_url'] = getattr(settings, 'CELERY_FLOWER_URL', None) # recent changes recent_changes = int(request.GET.get('changes', 50)) context['recent_changes'] = get_recent_changes(get_db(), recent_changes) context['rabbitmq_url'] = get_rabbitmq_management_url() context['hide_filters'] = True context['current_system'] = socket.gethostname() context['last_deploy'] = HqDeploy.get_latest(environment) context['snapshot'] = context['last_deploy'].code_snapshot if context['last_deploy'] else {} context.update(check_redis()) context.update(check_rabbitmq()) context.update(check_celery_health()) context.update(check_memcached()) context.update(check_es_cluster_health()) return render(request, "hqadmin/system_info.html", context)