def update_calculated_properties():
es = get_es()
q = {"filter": {"and": [
{"term": {"doc_type": "Domain"}},
{"term": {"is_snapshot": False}}
]}}
results = stream_es_query(q=q, es_url=ES_URLS["domains"], size=999999, chunksize=500, fields=["name"])
all_stats = _all_domain_stats()
for r in results:
dom = r["fields"]["name"]
calced_props = {
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": datetime.now().strftime(DATE_FORMAT),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties():
results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
calced_props = {
"_id": r["_id"],
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": json_format_datetime(datetime.utcnow()),
"cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
send_to_elasticsearch("domains", calced_props)
def update_calculated_properties():
results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
calced_props = {
"_id": r["_id"],
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": json_format_datetime(datetime.utcnow()),
"cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
send_to_elasticsearch("domains", calced_props)
def update_calculated_properties():
es = get_es()
q = {
"filter": {
"and": [{
"term": {
"doc_type": "Domain"
}
}, {
"term": {
"is_snapshot": False
}
}]
}
}
results = stream_es_query(q=q,
es_url=ES_URLS["domains"],
size=999999,
chunksize=500,
fields=["name"])
all_stats = _all_domain_stats()
for r in results:
dom = r["fields"]["name"]
calced_props = {
"cp_n_web_users":
int(all_stats["web_users"][dom]),
"cp_n_active_cc_users":
int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users":
int(all_stats["commcare_users"][dom]),
"cp_n_active_cases":
int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form":
total_distinct_users([dom]),
"cp_n_inactive_cases":
int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases":
int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases":
int(all_stats["cases"][dom]),
"cp_n_forms":
int(all_stats["forms"][dom]),
"cp_first_form":
CALC_FNS["first_form_submission"](dom, False),
"cp_last_form":
CALC_FNS["last_form_submission"](dom, False),
"cp_is_active":
CALC_FNS["active"](dom),
"cp_has_app":
CALC_FNS["has_app"](dom),
"cp_last_updated":
datetime.now().strftime(DATE_FORMAT),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]),
data={"doc": calced_props})
def update_calculated_properties():
results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
try:
calced_props = {
"_id": r["_id"],
"cp_n_web_users": int(all_stats["web_users"].get(dom, 0)),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"].get(dom, 0)),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_30_day_cases": int(CALC_FNS["cases_in_last"](dom, 30)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_90_day_cases": int(CALC_FNS["cases_in_last"](dom, 90)),
"cp_n_cases": int(all_stats["cases"].get(dom, 0)),
"cp_n_forms": int(all_stats["forms"].get(dom, 0)),
"cp_n_forms_30_d": int(CALC_FNS["forms_in_last"](dom, 30)),
"cp_n_forms_60_d": int(CALC_FNS["forms_in_last"](dom, 60)),
"cp_n_forms_90_d": int(CALC_FNS["forms_in_last"](dom, 90)),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": json_format_datetime(datetime.utcnow()),
"cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_n_sms_60_d": int(CALC_FNS["sms_in_last"](dom, 60)),
"cp_n_sms_90_d": int(CALC_FNS["sms_in_last"](dom, 90)),
"cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)),
"cp_n_sms_in_30_d": int(CALC_FNS["sms_in_in_last"](dom, 30)),
"cp_n_sms_in_60_d": int(CALC_FNS["sms_in_in_last"](dom, 60)),
"cp_n_sms_in_90_d": int(CALC_FNS["sms_in_in_last"](dom, 90)),
"cp_n_sms_out_30_d": int(CALC_FNS["sms_out_in_last"](dom, 30)),
"cp_n_sms_out_60_d": int(CALC_FNS["sms_out_in_last"](dom, 60)),
"cp_n_sms_out_90_d": int(CALC_FNS["sms_out_in_last"](dom, 90)),
"cp_n_j2me_30_d": int(CALC_FNS["j2me_forms_in_last"](dom, 30)),
"cp_n_j2me_60_d": int(CALC_FNS["j2me_forms_in_last"](dom, 60)),
"cp_n_j2me_90_d": int(CALC_FNS["j2me_forms_in_last"](dom, 90)),
"cp_j2me_90_d_bool": int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)),
"cp_300th_form": CALC_FNS["300th_form_submission"](dom)
}
if calced_props['cp_first_form'] is None:
del calced_props['cp_first_form']
if calced_props['cp_last_form'] is None:
del calced_props['cp_last_form']
if calced_props['cp_300th_form'] is None:
del calced_props['cp_300th_form']
send_to_elasticsearch("domains", calced_props)
except Exception, e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def update_calculated_properties():
results = DomainES().fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
try:
calced_props = {
"_id":
r["_id"],
"cp_n_web_users":
int(all_stats["web_users"].get(dom, 0)),
"cp_n_active_cc_users":
int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users":
int(all_stats["commcare_users"].get(dom, 0)),
"cp_n_active_cases":
int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form":
total_distinct_users([dom]),
"cp_n_inactive_cases":
int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_30_day_cases":
int(CALC_FNS["cases_in_last"](dom, 30)),
"cp_n_60_day_cases":
int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_90_day_cases":
int(CALC_FNS["cases_in_last"](dom, 90)),
"cp_n_cases":
int(all_stats["cases"].get(dom, 0)),
"cp_n_forms":
int(all_stats["forms"].get(dom, 0)),
"cp_n_forms_30_d":
int(CALC_FNS["forms_in_last"](dom, 30)),
"cp_n_forms_60_d":
int(CALC_FNS["forms_in_last"](dom, 60)),
"cp_n_forms_90_d":
int(CALC_FNS["forms_in_last"](dom, 90)),
"cp_first_form":
CALC_FNS["first_form_submission"](dom, False),
"cp_last_form":
CALC_FNS["last_form_submission"](dom, False),
"cp_is_active":
CALC_FNS["active"](dom),
"cp_has_app":
CALC_FNS["has_app"](dom),
"cp_last_updated":
json_format_datetime(datetime.utcnow()),
"cp_n_in_sms":
int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms":
int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever":
int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d":
int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_n_sms_60_d":
int(CALC_FNS["sms_in_last"](dom, 60)),
"cp_n_sms_90_d":
int(CALC_FNS["sms_in_last"](dom, 90)),
"cp_sms_ever":
int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d":
int(CALC_FNS["sms_in_last_bool"](dom, 30)),
"cp_n_sms_in_30_d":
int(CALC_FNS["sms_in_in_last"](dom, 30)),
"cp_n_sms_in_60_d":
int(CALC_FNS["sms_in_in_last"](dom, 60)),
"cp_n_sms_in_90_d":
int(CALC_FNS["sms_in_in_last"](dom, 90)),
"cp_n_sms_out_30_d":
int(CALC_FNS["sms_out_in_last"](dom, 30)),
"cp_n_sms_out_60_d":
int(CALC_FNS["sms_out_in_last"](dom, 60)),
"cp_n_sms_out_90_d":
int(CALC_FNS["sms_out_in_last"](dom, 90)),
"cp_n_j2me_30_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 30)),
"cp_n_j2me_60_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 60)),
"cp_n_j2me_90_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 90)),
"cp_j2me_90_d_bool":
int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)),
"cp_300th_form":
CALC_FNS["300th_form_submission"](dom)
}
if calced_props['cp_first_form'] is None:
del calced_props['cp_first_form']
if calced_props['cp_last_form'] is None:
del calced_props['cp_last_form']
if calced_props['cp_300th_form'] is None:
del calced_props['cp_300th_form']
send_to_elasticsearch("domains", calced_props)
except Exception, e:
notify_exception(
None,
message='Domain {} failed on stats calculations with {}'.
format(dom, e))
