def rows(self):
def numcell(text, value=None):
if value is None:
try:
value = int(text)
if math.isnan(value):
text = '---'
except ValueError:
value = text
return util.format_datatables_data(text=text, sort_key=value)
all_stats = _all_domain_stats()
domains = self.get_domains()
for domain in domains:
dom = getattr(domain, 'name', domain) # get the domain name if domains is a list of domain objects
yield [
getattr(domain, 'hr_name', dom), # get the hr_name if domain is a domain object
numcell(dom_calc("mobile_users", dom)),
numcell(all_stats["commcare_users"][dom]),
numcell(dom_calc("cases_in_last", dom, 120)),
numcell(all_stats["cases"][dom]),
numcell(all_stats["forms"][dom]),
dom_calc("first_form_submission", dom),
dom_calc("last_form_submission", dom),
numcell(all_stats["web_users"][dom]),
]
def update_calculated_properties():
es = get_es()
q = {"filter": {"and": [
{"term": {"doc_type": "Domain"}},
{"term": {"is_snapshot": False}}
]}}
results = stream_es_query(q=q, es_url=ES_URLS["domains"], size=999999, chunksize=500, fields=["name"])
all_stats = _all_domain_stats()
for r in results:
dom = r["fields"]["name"]
calced_props = {
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": datetime.now().strftime(DATE_FORMAT),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties():
es = get_es()
#todo: use some sort of ES scrolling/paginating
results = es.get(DOMAIN_INDEX + "/hqdomain/_search", data={"size": 99999})['hits']['hits']
all_stats = _all_domain_stats()
for r in results:
dom = r["_source"]["name"]
calced_props = {
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]), data={"doc": calced_props})
def update_calculated_properties():
results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
calced_props = {
"_id": r["_id"],
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": json_format_datetime(datetime.utcnow()),
"cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
send_to_elasticsearch("domains", calced_props)
def update_calculated_properties():
es = get_es()
#todo: use some sort of ES scrolling/paginating
results = es.get(DOMAIN_INDEX + "/hqdomain/_search",
data={"size": 99999})['hits']['hits']
all_stats = _all_domain_stats()
for r in results:
dom = r["_source"]["name"]
calced_props = {
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]),
data={"doc": calced_props})
def update_calculated_properties():
results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
calced_props = {
"_id": r["_id"],
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": json_format_datetime(datetime.utcnow()),
"cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
send_to_elasticsearch("domains", calced_props)
def update_calculated_properties():
es = get_es()
q = {
"filter": {
"and": [{
"term": {
"doc_type": "Domain"
}
}, {
"term": {
"is_snapshot": False
}
}]
}
}
results = stream_es_query(q=q,
es_url=ES_URLS["domains"],
size=999999,
chunksize=500,
fields=["name"])
all_stats = _all_domain_stats()
for r in results:
dom = r["fields"]["name"]
calced_props = {
"cp_n_web_users":
int(all_stats["web_users"][dom]),
"cp_n_active_cc_users":
int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users":
int(all_stats["commcare_users"][dom]),
"cp_n_active_cases":
int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form":
total_distinct_users([dom]),
"cp_n_inactive_cases":
int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases":
int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases":
int(all_stats["cases"][dom]),
"cp_n_forms":
int(all_stats["forms"][dom]),
"cp_first_form":
CALC_FNS["first_form_submission"](dom, False),
"cp_last_form":
CALC_FNS["last_form_submission"](dom, False),
"cp_is_active":
CALC_FNS["active"](dom),
"cp_has_app":
CALC_FNS["has_app"](dom),
"cp_last_updated":
datetime.now().strftime(DATE_FORMAT),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
es.post("%s/hqdomain/%s/_update" % (DOMAIN_INDEX, r["_id"]),
data={"doc": calced_props})
def update_calculated_properties():
results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
try:
calced_props = {
"_id": r["_id"],
"cp_n_web_users": int(all_stats["web_users"].get(dom, 0)),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"].get(dom, 0)),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_30_day_cases": int(CALC_FNS["cases_in_last"](dom, 30)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_90_day_cases": int(CALC_FNS["cases_in_last"](dom, 90)),
"cp_n_cases": int(all_stats["cases"].get(dom, 0)),
"cp_n_forms": int(all_stats["forms"].get(dom, 0)),
"cp_n_forms_30_d": int(CALC_FNS["forms_in_last"](dom, 30)),
"cp_n_forms_60_d": int(CALC_FNS["forms_in_last"](dom, 60)),
"cp_n_forms_90_d": int(CALC_FNS["forms_in_last"](dom, 90)),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": json_format_datetime(datetime.utcnow()),
"cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_n_sms_60_d": int(CALC_FNS["sms_in_last"](dom, 60)),
"cp_n_sms_90_d": int(CALC_FNS["sms_in_last"](dom, 90)),
"cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)),
"cp_n_sms_in_30_d": int(CALC_FNS["sms_in_in_last"](dom, 30)),
"cp_n_sms_in_60_d": int(CALC_FNS["sms_in_in_last"](dom, 60)),
"cp_n_sms_in_90_d": int(CALC_FNS["sms_in_in_last"](dom, 90)),
"cp_n_sms_out_30_d": int(CALC_FNS["sms_out_in_last"](dom, 30)),
"cp_n_sms_out_60_d": int(CALC_FNS["sms_out_in_last"](dom, 60)),
"cp_n_sms_out_90_d": int(CALC_FNS["sms_out_in_last"](dom, 90)),
"cp_n_j2me_30_d": int(CALC_FNS["j2me_forms_in_last"](dom, 30)),
"cp_n_j2me_60_d": int(CALC_FNS["j2me_forms_in_last"](dom, 60)),
"cp_n_j2me_90_d": int(CALC_FNS["j2me_forms_in_last"](dom, 90)),
"cp_j2me_90_d_bool": int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)),
"cp_300th_form": CALC_FNS["300th_form_submission"](dom)
}
if calced_props['cp_first_form'] is None:
del calced_props['cp_first_form']
if calced_props['cp_last_form'] is None:
del calced_props['cp_last_form']
if calced_props['cp_300th_form'] is None:
del calced_props['cp_300th_form']
send_to_elasticsearch("domains", calced_props)
except Exception, e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
except Exception, e:
messages.warning(request, "Update for %s failed: %s" % (row.get("name", "<No Name>"), e))
fail_count += 1
if success_count:
messages.success(request, "%s domains successfully updated" % success_count)
if fail_count:
messages.error(request, "%s domains had errors. details above." % fail_count)
except Exception, e:
messages.error(request, "Something went wrong! Update failed. Here's your error: %s" % e)
# one wonders if this will eventually have to paginate
domains = Domain.get_all()
from corehq.apps.domain.calculations import _all_domain_stats
all_stats = _all_domain_stats()
for dom in domains:
dom.web_users = int(all_stats["web_users"][dom.name])
dom.commcare_users = int(all_stats["commcare_users"][dom.name])
dom.cases = int(all_stats["cases"][dom.name])
dom.forms = int(all_stats["forms"][dom.name])
if dom.forms:
try:
dom.first_submission = string_to_datetime(
XFormInstance.get_db()
.view(
"receiverwrapper/all_submissions_by_domain",
reduce=False,
limit=1,
startkey=[dom.name, "by_date"],
endkey=[dom.name, "by_date", {}],
request, "%s domains successfully updated" % success_count)
if fail_count:
messages.error(
request,
"%s domains had errors. details above." % fail_count)
except Exception, e:
messages.error(
request,
"Something went wrong! Update failed. Here's your error: %s" %
e)
# one wonders if this will eventually have to paginate
domains = Domain.get_all()
from corehq.apps.domain.calculations import _all_domain_stats
all_stats = _all_domain_stats()
for dom in domains:
dom.web_users = int(all_stats["web_users"][dom.name])
dom.commcare_users = int(all_stats["commcare_users"][dom.name])
dom.cases = int(all_stats["cases"][dom.name])
dom.forms = int(all_stats["forms"][dom.name])
if dom.forms:
try:
dom.first_submission = string_to_datetime(XFormInstance.get_db().view\
("couchforms/all_submissions_by_domain",
reduce=False, limit=1,
startkey=[dom.name, "by_date"],
endkey=[dom.name, "by_date", {}]).all()[0]["key"][2]).strftime("%Y-%m-%d")
except Exception:
dom.first_submission = ""
def update_calculated_properties():
results = DomainES().fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
try:
calced_props = {
"_id":
r["_id"],
"cp_n_web_users":
int(all_stats["web_users"].get(dom, 0)),
"cp_n_active_cc_users":
int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users":
int(all_stats["commcare_users"].get(dom, 0)),
"cp_n_active_cases":
int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form":
total_distinct_users([dom]),
"cp_n_inactive_cases":
int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_30_day_cases":
int(CALC_FNS["cases_in_last"](dom, 30)),
"cp_n_60_day_cases":
int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_90_day_cases":
int(CALC_FNS["cases_in_last"](dom, 90)),
"cp_n_cases":
int(all_stats["cases"].get(dom, 0)),
"cp_n_forms":
int(all_stats["forms"].get(dom, 0)),
"cp_n_forms_30_d":
int(CALC_FNS["forms_in_last"](dom, 30)),
"cp_n_forms_60_d":
int(CALC_FNS["forms_in_last"](dom, 60)),
"cp_n_forms_90_d":
int(CALC_FNS["forms_in_last"](dom, 90)),
"cp_first_form":
CALC_FNS["first_form_submission"](dom, False),
"cp_last_form":
CALC_FNS["last_form_submission"](dom, False),
"cp_is_active":
CALC_FNS["active"](dom),
"cp_has_app":
CALC_FNS["has_app"](dom),
"cp_last_updated":
json_format_datetime(datetime.utcnow()),
"cp_n_in_sms":
int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms":
int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever":
int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d":
int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_n_sms_60_d":
int(CALC_FNS["sms_in_last"](dom, 60)),
"cp_n_sms_90_d":
int(CALC_FNS["sms_in_last"](dom, 90)),
"cp_sms_ever":
int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d":
int(CALC_FNS["sms_in_last_bool"](dom, 30)),
"cp_n_sms_in_30_d":
int(CALC_FNS["sms_in_in_last"](dom, 30)),
"cp_n_sms_in_60_d":
int(CALC_FNS["sms_in_in_last"](dom, 60)),
"cp_n_sms_in_90_d":
int(CALC_FNS["sms_in_in_last"](dom, 90)),
"cp_n_sms_out_30_d":
int(CALC_FNS["sms_out_in_last"](dom, 30)),
"cp_n_sms_out_60_d":
int(CALC_FNS["sms_out_in_last"](dom, 60)),
"cp_n_sms_out_90_d":
int(CALC_FNS["sms_out_in_last"](dom, 90)),
"cp_n_j2me_30_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 30)),
"cp_n_j2me_60_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 60)),
"cp_n_j2me_90_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 90)),
"cp_j2me_90_d_bool":
int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)),
"cp_300th_form":
CALC_FNS["300th_form_submission"](dom)
}
if calced_props['cp_first_form'] is None:
del calced_props['cp_first_form']
if calced_props['cp_last_form'] is None:
del calced_props['cp_last_form']
if calced_props['cp_300th_form'] is None:
del calced_props['cp_300th_form']
send_to_elasticsearch("domains", calced_props)
except Exception, e:
notify_exception(
None,
message='Domain {} failed on stats calculations with {}'.
format(dom, e))
