def _update_calculated_properties():
results = DomainES().filter(
get_domains_to_update_es_filter()
).fields(["name", "_id"]).run().hits
all_stats = all_domain_stats()
active_users_by_domain = {}
for r in results:
dom = r["name"]
domain_obj = Domain.get_by_name(dom)
if not domain_obj:
send_to_elasticsearch("domains", r, delete=True)
continue
try:
props = calced_props(domain_obj, r["_id"], all_stats)
active_users_by_domain[dom] = props['cp_n_active_cc_users']
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
datadog_report_user_stats('commcare.active_mobile_workers.count', active_users_by_domain)
def run_datadog_user_stats():
all_stats = all_domain_stats()
datadog_report_user_stats(
'commcare.mobile_workers.count',
commcare_users_by_domain=all_stats['commcare_users'],
)
def update_calculated_properties():
results = DomainES().filter(get_domains_to_update_es_filter()).fields(
["name", "_id", "cp_last_updated"]).scroll()
all_stats = all_domain_stats()
for r in results:
dom = r["name"]
domain_obj = Domain.get_by_name(dom)
if not domain_obj:
send_to_elasticsearch("domains", r, delete=True)
continue
try:
props = calced_props(domain_obj, r["_id"], all_stats)
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(
None,
message='Domain {} failed on stats calculations with {}'.
format(dom, e))
def update_calculated_properties_for_domains(domains):
"""
:param domains: list of {'name': <name>, '_id': <id>} entries
"""
# relying on caching for efficiency
all_stats = all_domain_stats()
active_users_by_domain = {}
for domain in domains:
domain_obj = Domain.get_by_name(domain['name'])
if not domain_obj:
send_to_elasticsearch("domains", domain, delete=True)
continue
try:
props = calced_props(domain_obj, domain['_id'], all_stats)
active_users_by_domain[domain['name']] = props['cp_n_active_cc_users']
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(
None, message='Domain {} failed on stats calculations with {}'.format(domain['name'], e)
)
datadog_report_user_stats('commcare.active_mobile_workers.count', active_users_by_domain)
def update_calculated_properties():
results = DomainES().fields(["name", "_id", "cp_last_updated"]).scroll()
all_stats = all_domain_stats()
for r in results:
dom = r["name"]
try:
last_form_submission = CALC_FNS["last_form_submission"](dom, False)
if _skip_updating_domain_stats(r.get("cp_last_updated"), last_form_submission):
continue
props = calced_props(dom, r["_id"], all_stats)
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def update_calculated_properties():
results = DomainES().fields(["name", "_id", "cp_last_updated"]).scroll()
all_stats = all_domain_stats()
for r in results:
dom = r["name"]
try:
last_form_submission = CALC_FNS["last_form_submission"](dom, False)
if _skip_updating_domain_stats(r.get("cp_last_updated"), last_form_submission):
continue
props = calced_props(dom, r["_id"], all_stats)
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props)
except Exception, e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def update_calculated_properties():
results = DomainES().filter(
get_domains_to_update_es_filter()
).fields(["name", "_id"]).run().hits
all_stats = all_domain_stats()
for r in results:
dom = r["name"]
domain_obj = Domain.get_by_name(dom)
if not domain_obj:
send_to_elasticsearch("domains", r, delete=True)
continue
try:
props = calced_props(domain_obj, r["_id"], all_stats)
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def test_domain_does_not_have_apps(self):
sms_doc = self.create_sms_in_es(self.domain.name, INCOMING)
self.addCleanup(self.delete_sms_in_es, sms_doc)
all_stats = all_domain_stats()
props = calced_props(self.domain, self.domain._id, all_stats)
self.assertFalse(props['cp_has_app'])
def test_calculated_properties_are_serializable(self):
sms_doc = self.create_sms_in_es(self.domain.name, INCOMING)
self.addCleanup(self.delete_sms_in_es, sms_doc)
all_stats = all_domain_stats()
props = calced_props(self.domain, self.domain._id, all_stats)
json.dumps(props)
def test_sanity(self):
all_stats = all_domain_stats()
props = calced_props(self.domain, self.domain._id, all_stats)
self.assertFalse(props['cp_has_app'])
# ensure serializable
json.dumps(props)
def test_sanity(self):
all_stats = all_domain_stats()
props = calced_props(self.domain, self.domain._id, all_stats)
self.assertFalse(props['cp_has_app'])
# ensure serializable
json.dumps(props)