def update_calculated_properties(): es = get_es() q = {"filter": {"and": [ {"term": {"doc_type": "Domain"}}, {"term": {"is_snapshot": False}} ]}} results = stream_es_query(q=q, es_url=ES_URLS["domains"], size=999999, chunksize=500, fields=["name"]) all_stats = _all_domain_stats() for r in results: dom = r["fields"]["name"] calced_props = { "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": datetime.now().strftime(DATE_FORMAT), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties(): results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] send_to_elasticsearch("domains", calced_props)
def update_calculated_properties(): es = get_es() q = { "filter": { "and": [{ "term": { "doc_type": "Domain" } }, { "term": { "is_snapshot": False } }] } } results = stream_es_query(q=q, es_url=ES_URLS["domains"], size=999999, chunksize=500, fields=["name"]) all_stats = _all_domain_stats() for r in results: dom = r["fields"]["name"] calced_props = { "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": datetime.now().strftime(DATE_FORMAT), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties(): results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] try: calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"].get(dom, 0)), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"].get(dom, 0)), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_30_day_cases": int(CALC_FNS["cases_in_last"](dom, 30)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_90_day_cases": int(CALC_FNS["cases_in_last"](dom, 90)), "cp_n_cases": int(all_stats["cases"].get(dom, 0)), "cp_n_forms": int(all_stats["forms"].get(dom, 0)), "cp_n_forms_30_d": int(CALC_FNS["forms_in_last"](dom, 30)), "cp_n_forms_60_d": int(CALC_FNS["forms_in_last"](dom, 60)), "cp_n_forms_90_d": int(CALC_FNS["forms_in_last"](dom, 90)), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_n_sms_60_d": int(CALC_FNS["sms_in_last"](dom, 60)), "cp_n_sms_90_d": int(CALC_FNS["sms_in_last"](dom, 90)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), "cp_n_sms_in_30_d": int(CALC_FNS["sms_in_in_last"](dom, 30)), "cp_n_sms_in_60_d": int(CALC_FNS["sms_in_in_last"](dom, 60)), "cp_n_sms_in_90_d": int(CALC_FNS["sms_in_in_last"](dom, 90)), "cp_n_sms_out_30_d": int(CALC_FNS["sms_out_in_last"](dom, 30)), "cp_n_sms_out_60_d": int(CALC_FNS["sms_out_in_last"](dom, 60)), "cp_n_sms_out_90_d": int(CALC_FNS["sms_out_in_last"](dom, 90)), "cp_n_j2me_30_d": int(CALC_FNS["j2me_forms_in_last"](dom, 30)), "cp_n_j2me_60_d": int(CALC_FNS["j2me_forms_in_last"](dom, 60)), "cp_n_j2me_90_d": int(CALC_FNS["j2me_forms_in_last"](dom, 90)), "cp_j2me_90_d_bool": int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)), "cp_300th_form": CALC_FNS["300th_form_submission"](dom) } if calced_props['cp_first_form'] is None: del calced_props['cp_first_form'] if calced_props['cp_last_form'] is None: del calced_props['cp_last_form'] if calced_props['cp_300th_form'] is None: del calced_props['cp_300th_form'] send_to_elasticsearch("domains", calced_props) except Exception, e: notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def update_calculated_properties(): results = DomainES().fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] try: calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"].get(dom, 0)), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"].get(dom, 0)), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_30_day_cases": int(CALC_FNS["cases_in_last"](dom, 30)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_90_day_cases": int(CALC_FNS["cases_in_last"](dom, 90)), "cp_n_cases": int(all_stats["cases"].get(dom, 0)), "cp_n_forms": int(all_stats["forms"].get(dom, 0)), "cp_n_forms_30_d": int(CALC_FNS["forms_in_last"](dom, 30)), "cp_n_forms_60_d": int(CALC_FNS["forms_in_last"](dom, 60)), "cp_n_forms_90_d": int(CALC_FNS["forms_in_last"](dom, 90)), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_n_sms_60_d": int(CALC_FNS["sms_in_last"](dom, 60)), "cp_n_sms_90_d": int(CALC_FNS["sms_in_last"](dom, 90)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), "cp_n_sms_in_30_d": int(CALC_FNS["sms_in_in_last"](dom, 30)), "cp_n_sms_in_60_d": int(CALC_FNS["sms_in_in_last"](dom, 60)), "cp_n_sms_in_90_d": int(CALC_FNS["sms_in_in_last"](dom, 90)), "cp_n_sms_out_30_d": int(CALC_FNS["sms_out_in_last"](dom, 30)), "cp_n_sms_out_60_d": int(CALC_FNS["sms_out_in_last"](dom, 60)), "cp_n_sms_out_90_d": int(CALC_FNS["sms_out_in_last"](dom, 90)), "cp_n_j2me_30_d": int(CALC_FNS["j2me_forms_in_last"](dom, 30)), "cp_n_j2me_60_d": int(CALC_FNS["j2me_forms_in_last"](dom, 60)), "cp_n_j2me_90_d": int(CALC_FNS["j2me_forms_in_last"](dom, 90)), "cp_j2me_90_d_bool": int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)), "cp_300th_form": CALC_FNS["300th_form_submission"](dom) } if calced_props['cp_first_form'] is None: del calced_props['cp_first_form'] if calced_props['cp_last_form'] is None: del calced_props['cp_last_form'] if calced_props['cp_300th_form'] is None: del calced_props['cp_300th_form'] send_to_elasticsearch("domains", calced_props) except Exception, e: notify_exception( None, message='Domain {} failed on stats calculations with {}'. format(dom, e))