def rows(self): def numcell(text, value=None): if value is None: try: value = int(text) if math.isnan(value): text = '---' except ValueError: value = text return util.format_datatables_data(text=text, sort_key=value) all_stats = _all_domain_stats() domains = self.get_domains() for domain in domains: dom = getattr(domain, 'name', domain) # get the domain name if domains is a list of domain objects yield [ getattr(domain, 'hr_name', dom), # get the hr_name if domain is a domain object numcell(dom_calc("mobile_users", dom)), numcell(all_stats["commcare_users"][dom]), numcell(dom_calc("cases_in_last", dom, 120)), numcell(all_stats["cases"][dom]), numcell(all_stats["forms"][dom]), dom_calc("first_form_submission", dom), dom_calc("last_form_submission", dom), numcell(all_stats["web_users"][dom]), ]
def update_calculated_properties(): es = get_es() q = {"filter": {"and": [ {"term": {"doc_type": "Domain"}}, {"term": {"is_snapshot": False}} ]}} results = stream_es_query(q=q, es_url=ES_URLS["domains"], size=999999, chunksize=500, fields=["name"]) all_stats = _all_domain_stats() for r in results: dom = r["fields"]["name"] calced_props = { "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": datetime.now().strftime(DATE_FORMAT), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties(): es = get_es() #todo: use some sort of ES scrolling/paginating results = es.get(DOMAIN_INDEX + "/hqdomain/_search", data={"size": 99999})['hits']['hits'] all_stats = _all_domain_stats() for r in results: dom = r["_source"]["name"] calced_props = { "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties(): results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] send_to_elasticsearch("domains", calced_props)
def update_calculated_properties(): es = get_es() #todo: use some sort of ES scrolling/paginating results = es.get(DOMAIN_INDEX + "/hqdomain/_search", data={"size": 99999})['hits']['hits'] all_stats = _all_domain_stats() for r in results: dom = r["_source"]["name"] calced_props = { "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties(): es = get_es() q = { "filter": { "and": [{ "term": { "doc_type": "Domain" } }, { "term": { "is_snapshot": False } }] } } results = stream_es_query(q=q, es_url=ES_URLS["domains"], size=999999, chunksize=500, fields=["name"]) all_stats = _all_domain_stats() for r in results: dom = r["fields"]["name"] calced_props = { "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": datetime.now().strftime(DATE_FORMAT), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties(): results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] try: calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"].get(dom, 0)), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"].get(dom, 0)), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_30_day_cases": int(CALC_FNS["cases_in_last"](dom, 30)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_90_day_cases": int(CALC_FNS["cases_in_last"](dom, 90)), "cp_n_cases": int(all_stats["cases"].get(dom, 0)), "cp_n_forms": int(all_stats["forms"].get(dom, 0)), "cp_n_forms_30_d": int(CALC_FNS["forms_in_last"](dom, 30)), "cp_n_forms_60_d": int(CALC_FNS["forms_in_last"](dom, 60)), "cp_n_forms_90_d": int(CALC_FNS["forms_in_last"](dom, 90)), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_n_sms_60_d": int(CALC_FNS["sms_in_last"](dom, 60)), "cp_n_sms_90_d": int(CALC_FNS["sms_in_last"](dom, 90)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), "cp_n_sms_in_30_d": int(CALC_FNS["sms_in_in_last"](dom, 30)), "cp_n_sms_in_60_d": int(CALC_FNS["sms_in_in_last"](dom, 60)), "cp_n_sms_in_90_d": int(CALC_FNS["sms_in_in_last"](dom, 90)), "cp_n_sms_out_30_d": int(CALC_FNS["sms_out_in_last"](dom, 30)), "cp_n_sms_out_60_d": int(CALC_FNS["sms_out_in_last"](dom, 60)), "cp_n_sms_out_90_d": int(CALC_FNS["sms_out_in_last"](dom, 90)), "cp_n_j2me_30_d": int(CALC_FNS["j2me_forms_in_last"](dom, 30)), "cp_n_j2me_60_d": int(CALC_FNS["j2me_forms_in_last"](dom, 60)), "cp_n_j2me_90_d": int(CALC_FNS["j2me_forms_in_last"](dom, 90)), "cp_j2me_90_d_bool": int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)), "cp_300th_form": CALC_FNS["300th_form_submission"](dom) } if calced_props['cp_first_form'] is None: del calced_props['cp_first_form'] if calced_props['cp_last_form'] is None: del calced_props['cp_last_form'] if calced_props['cp_300th_form'] is None: del calced_props['cp_300th_form'] send_to_elasticsearch("domains", calced_props) except Exception, e: notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
except Exception, e: messages.warning(request, "Update for %s failed: %s" % (row.get("name", "<No Name>"), e)) fail_count += 1 if success_count: messages.success(request, "%s domains successfully updated" % success_count) if fail_count: messages.error(request, "%s domains had errors. details above." % fail_count) except Exception, e: messages.error(request, "Something went wrong! Update failed. Here's your error: %s" % e) # one wonders if this will eventually have to paginate domains = Domain.get_all() from corehq.apps.domain.calculations import _all_domain_stats all_stats = _all_domain_stats() for dom in domains: dom.web_users = int(all_stats["web_users"][dom.name]) dom.commcare_users = int(all_stats["commcare_users"][dom.name]) dom.cases = int(all_stats["cases"][dom.name]) dom.forms = int(all_stats["forms"][dom.name]) if dom.forms: try: dom.first_submission = string_to_datetime( XFormInstance.get_db() .view( "receiverwrapper/all_submissions_by_domain", reduce=False, limit=1, startkey=[dom.name, "by_date"], endkey=[dom.name, "by_date", {}],
request, "%s domains successfully updated" % success_count) if fail_count: messages.error( request, "%s domains had errors. details above." % fail_count) except Exception, e: messages.error( request, "Something went wrong! Update failed. Here's your error: %s" % e) # one wonders if this will eventually have to paginate domains = Domain.get_all() from corehq.apps.domain.calculations import _all_domain_stats all_stats = _all_domain_stats() for dom in domains: dom.web_users = int(all_stats["web_users"][dom.name]) dom.commcare_users = int(all_stats["commcare_users"][dom.name]) dom.cases = int(all_stats["cases"][dom.name]) dom.forms = int(all_stats["forms"][dom.name]) if dom.forms: try: dom.first_submission = string_to_datetime(XFormInstance.get_db().view\ ("couchforms/all_submissions_by_domain", reduce=False, limit=1, startkey=[dom.name, "by_date"], endkey=[dom.name, "by_date", {}]).all()[0]["key"][2]).strftime("%Y-%m-%d") except Exception: dom.first_submission = ""
def update_calculated_properties(): results = DomainES().fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] try: calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"].get(dom, 0)), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"].get(dom, 0)), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_30_day_cases": int(CALC_FNS["cases_in_last"](dom, 30)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_90_day_cases": int(CALC_FNS["cases_in_last"](dom, 90)), "cp_n_cases": int(all_stats["cases"].get(dom, 0)), "cp_n_forms": int(all_stats["forms"].get(dom, 0)), "cp_n_forms_30_d": int(CALC_FNS["forms_in_last"](dom, 30)), "cp_n_forms_60_d": int(CALC_FNS["forms_in_last"](dom, 60)), "cp_n_forms_90_d": int(CALC_FNS["forms_in_last"](dom, 90)), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_n_sms_60_d": int(CALC_FNS["sms_in_last"](dom, 60)), "cp_n_sms_90_d": int(CALC_FNS["sms_in_last"](dom, 90)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), "cp_n_sms_in_30_d": int(CALC_FNS["sms_in_in_last"](dom, 30)), "cp_n_sms_in_60_d": int(CALC_FNS["sms_in_in_last"](dom, 60)), "cp_n_sms_in_90_d": int(CALC_FNS["sms_in_in_last"](dom, 90)), "cp_n_sms_out_30_d": int(CALC_FNS["sms_out_in_last"](dom, 30)), "cp_n_sms_out_60_d": int(CALC_FNS["sms_out_in_last"](dom, 60)), "cp_n_sms_out_90_d": int(CALC_FNS["sms_out_in_last"](dom, 90)), "cp_n_j2me_30_d": int(CALC_FNS["j2me_forms_in_last"](dom, 30)), "cp_n_j2me_60_d": int(CALC_FNS["j2me_forms_in_last"](dom, 60)), "cp_n_j2me_90_d": int(CALC_FNS["j2me_forms_in_last"](dom, 90)), "cp_j2me_90_d_bool": int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)), "cp_300th_form": CALC_FNS["300th_form_submission"](dom) } if calced_props['cp_first_form'] is None: del calced_props['cp_first_form'] if calced_props['cp_last_form'] is None: del calced_props['cp_last_form'] if calced_props['cp_300th_form'] is None: del calced_props['cp_300th_form'] send_to_elasticsearch("domains", calced_props) except Exception, e: notify_exception( None, message='Domain {} failed on stats calculations with {}'. format(dom, e))