def get_active_countries_stats_data(domains, datespan, interval, datefield="received_on"):
"""
Returns list of timestamps and how many countries were active in the 30
days before the timestamp
"""
histo_data = []
for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
t = timestamp
f = timestamp - relativedelta(days=30)
form_query = (
FormES()
.domain(domains)
.terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND)
.submitted(gte=f, lte=t)
.size(0)
)
domains = form_query.run().facet("domains", "terms")
domains = [x["term"] for x in domains]
countries = DomainES().in_domains(domains).terms_facet("countries", "countries", size=COUNTRY_COUNT_UPPER_BOUND)
c = len(countries.run().facet("countries", "terms"))
if c > 0:
histo_data.append(get_data_point(c, timestamp))
return format_return_data(histo_data, 0, datespan)
def handle(self, *args, **options):
# Find all apps using vellum case management
app_query = AppES().is_build(False).term(
'vellum_case_management', True).source(['domain', '_id'])
# Find all domains created after vellum case management was released,
# since their apps were never in the old world and therefore never migrated
epoch = datetime(year=2016, month=7, day=15)
domain_query = DomainES().date_range('date_created',
gt=epoch).source('name')
new_domains = {d['name']: 1 for d in domain_query.run().hits}
hits = app_query.run().hits
for hit in hits:
if hit['domain'] not in new_domains:
try:
app = get_app(hit['domain'], hit['_id'])
modules = [
m for m in app.modules if m.module_type == 'basic'
]
for module in modules:
forms = [
f for f in module.forms if f.doc_type == 'Form'
]
for form in forms:
if form.requires != 'case' and 'load' in form.case_references:
if form.case_references['load']:
logger.info(
'{} is suspicious: /a/{}/apps/view/{}/modules-{}/forms-{}'
.format(form.unique_id, app.domain,
app.id, module.id, form.id))
except Http404:
pass
logger.info('done with cmitfb_identify_broken_forms')
def get_commconnect_domain_stats_data(domains, datespan, interval, datefield="date_created", additional_params_es={}):
"""
Returns domains that have used SMS.
Returned based on date domain is created
"""
sms = SMSES().domain(domains).terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND).size(0)
if additional_params_es:
sms = add_params_to_query(sms, additional_params_es)
sms_domains = {x["term"] for x in sms.run().facet("domains", "terms")}
domains_after_date = (
DomainES()
.in_domains(sms_domains)
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.size(0)
)
histo_data = domains_after_date.run().facet("date", "entries")
domains_before_date = DomainES().in_domains(sms_domains).created(lt=datespan.startdate).size(0)
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_commconnect_domain_stats_data(domains,
datespan,
interval,
datefield='date_created',
additional_params_es={}):
"""
Returns domains that have used SMS.
Returned based on date domain is created
"""
sms = (SMSES().domain(domains).terms_facet('domain',
'domains',
size=LARGE_ES_NUMBER).size(0))
if additional_params_es:
sms = add_params_to_query(sms, additional_params_es)
sms_domains = {x['term'] for x in sms.run().facet('domains', 'terms')}
domains_after_date = (DomainES().in_domains(sms_domains).created(
gte=datespan.startdate,
lte=datespan.enddate).date_histogram('date', datefield,
interval).size(0))
histo_data = domains_after_date.run().facet('date', 'entries')
domains_before_date = (DomainES().in_domains(sms_domains).created(
lt=datespan.startdate).size(0))
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_sms_only_domain_stats_data(domains,
datespan,
interval,
datefield='date_created'):
"""
Returns domains that have only used SMS and not forms.
Returned based on date domain is created
"""
histo_data = []
sms = (SMSES().domain(domains).terms_facet('domain',
'domains',
size=LARGE_ES_NUMBER).size(0))
forms = (FormES().domain(domains).terms_facet(
'domain', 'domains', size=LARGE_ES_NUMBER).size(0))
sms_domains = {x['term'] for x in sms.run().facet('domains', 'terms')}
form_domains = {x['term'] for x in forms.run().facet('domains', 'terms')}
sms_only_domains = sms_domains - form_domains
domains_after_date = (DomainES().in_domains(sms_only_domains).created(
gte=datespan.startdate,
lte=datespan.enddate).date_histogram('date', datefield,
interval).size(0))
histo_data = domains_after_date.run().facet('date', 'entries')
domains_before_date = (DomainES().in_domains(sms_only_domains).created(
lt=datespan.startdate).size(0))
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_sms_only_domain_stats_data(domains, datespan, interval, datefield="date_created"):
"""
Returns domains that have only used SMS and not forms.
Returned based on date domain is created
"""
histo_data = []
sms = SMSES().domain(domains).terms_aggregation("domain", "domains").size(0)
forms = FormES().domain(domains).terms_aggregation("domain", "domains").size(0)
sms_domains = set(sms.run().aggregations.domains.keys)
form_domains = set(forms.run().aggregations.domains.keys)
sms_only_domains = sms_domains - form_domains
domains_after_date = (
DomainES()
.in_domains(sms_only_domains)
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.size(0)
)
histo_data = domains_after_date.run().aggregations.date.as_facet_result()
domains_before_date = DomainES().in_domains(sms_only_domains).created(lt=datespan.startdate).size(0)
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_commconnect_domain_stats_data(domains, datespan, interval, datefield="date_created", additional_params_es={}):
"""
Returns domains that have used SMS.
Returned based on date domain is created
"""
sms = SMSES().domain(domains).terms_aggregation("domain", "domains").size(0)
if additional_params_es:
sms = add_params_to_query(sms, additional_params_es)
sms_domains = set(sms.run().aggregations.domains.keys)
domains_after_date = (
DomainES()
.in_domains(sms_domains)
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.size(0)
)
histo_data = domains_after_date.run().aggregations.date.as_facet_result()
domains_before_date = DomainES().in_domains(sms_domains).created(lt=datespan.startdate).size(0)
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_sms_only_domain_stats_data(domains,
datespan,
interval,
datefield='date_created'):
"""
Returns domains that have only used SMS and not forms.
Returned based on date domain is created
"""
histo_data = []
sms = (SMSES().domain(domains).terms_aggregation('domain',
'domains').size(0))
forms = (FormES().domain(domains).terms_aggregation('domain',
'domains').size(0))
sms_domains = set(sms.run().aggregations.domains.keys)
form_domains = set(forms.run().aggregations.domains.keys)
sms_only_domains = sms_domains - form_domains
domains_after_date = (DomainES().in_domains(sms_only_domains).created(
gte=datespan.startdate,
lte=datespan.enddate).date_histogram('date', datefield,
interval).size(0))
histo_data = domains_after_date.run().aggregations.date.as_facet_result()
domains_before_date = (DomainES().in_domains(sms_only_domains).created(
lt=datespan.startdate).size(0))
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_commconnect_domain_stats_data(domains,
datespan,
interval,
datefield='date_created',
additional_params_es={}):
"""
Returns domains that have used SMS.
Returned based on date domain is created
"""
sms = (SMSES().domain(domains).terms_aggregation('domain',
'domains').size(0))
if additional_params_es:
sms = add_params_to_query(sms, additional_params_es)
sms_domains = set(sms.run().aggregations.domains.keys)
domains_after_date = (DomainES().in_domains(sms_domains).created(
gte=datespan.startdate,
lte=datespan.enddate).date_histogram('date', datefield,
interval).size(0))
histo_data = domains_after_date.run().aggregations.date.as_facet_result()
domains_before_date = (DomainES().in_domains(sms_domains).created(
lt=datespan.startdate).size(0))
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_sms_only_domain_stats_data(domains, datespan, interval, datefield="date_created"):
"""
Returns domains that have only used SMS and not forms.
Returned based on date domain is created
"""
histo_data = []
sms = SMSES().domain(domains).terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND).size(0)
forms = FormES().domain(domains).terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND).size(0)
sms_domains = {x["term"] for x in sms.run().facet("domains", "terms")}
form_domains = {x["term"] for x in forms.run().facet("domains", "terms")}
sms_only_domains = sms_domains - form_domains
domains_after_date = (
DomainES()
.in_domains(sms_only_domains)
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.size(0)
)
histo_data = domains_after_date.run().facet("date", "entries")
domains_before_date = DomainES().in_domains(sms_only_domains).created(lt=datespan.startdate).size(0)
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_project_spaces(facets=None):
"""
Returns a list of names of project spaces that satisfy the facets
"""
real_domain_query = DomainES().fields(["name"]).size(DOMAIN_COUNT_UPPER_BOUND)
if facets:
real_domain_query = add_params_to_query(real_domain_query, facets)
domain_names = [hit["name"] for hit in real_domain_query.run().hits]
if "search" in facets:
return list(set(domain_names) & set(facets["search"]))
return domain_names
def get_domain_stats_data(domains, datespan, interval, datefield="date_created"):
domains_after_date = (
DomainES()
.in_domains(domains)
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.size(0)
)
histo_data = domains_after_date.run().facet("date", "entries")
domains_before_date = DomainES().in_domains(domains).created(lt=datespan.startdate).size(0)
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def dehydrate_calculated_properties(self, bundle):
calc_prop_prefix = 'cp_'
domain_obj = _get_domain(bundle)
try:
es_data = (DomainES()
.in_domains([domain_obj.name])
.size(1)
.run()
.hits[0])
base_properties = {
prop_name: es_data[prop_name]
for prop_name in es_data
if prop_name.startswith(calc_prop_prefix)
}
try:
audit_record = DomainAuditRecordEntry.objects.get(domain=domain_obj.name)
except DomainAuditRecordEntry.DoesNotExist:
audit_record = None
extra_properties = {
field.name: getattr(audit_record, field.name, 0)
for field in DomainAuditRecordEntry._meta.fields
if field.name.startswith(calc_prop_prefix)
}
base_properties.update(extra_properties)
return base_properties
except IndexError:
logging.exception('Problem getting calculated properties for {}'.format(domain_obj.name))
return {}
def update_calculated_properties():
results = DomainES().filter(get_domains_to_update_es_filter()).fields(
["name", "_id", "cp_last_updated"]).scroll()
all_stats = all_domain_stats()
for r in results:
dom = r["name"]
domain_obj = Domain.get_by_name(dom)
if not domain_obj:
send_to_elasticsearch("domains", r, delete=True)
continue
try:
props = calced_props(domain_obj, r["_id"], all_stats)
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(
None,
message='Domain {} failed on stats calculations with {}'.
format(dom, e))
def get_domain_stats_data(domains,
datespan,
interval,
datefield="date_created"):
domains_after_date = (DomainES().in_domains(domains).created(
gte=datespan.startdate,
lte=datespan.enddate).date_histogram('date', datefield,
interval).size(0))
histo_data = domains_after_date.run().aggregations.date.as_facet_result()
domains_before_date = (DomainES().in_domains(domains).created(
lt=datespan.startdate).size(0))
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def get_active_countries_stats_data(domains,
datespan,
interval,
datefield='received_on'):
"""
Returns list of timestamps and how many countries were active in the 30
days before the timestamp
"""
histo_data = []
for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
t = timestamp
f = timestamp - relativedelta(days=30)
form_query = (FormES().domain(domains).terms_facet(
'domain', 'domains',
size=LARGE_ES_NUMBER).submitted(gte=f, lte=t).size(0))
domains = form_query.run().facet('domains', "terms")
domains = [x['term'] for x in domains]
countries = (DomainES().in_domains(domains).terms_facet(
'countries', 'countries', size=LARGE_ES_NUMBER))
c = len(countries.run().facet('countries', 'terms'))
if c > 0:
histo_data.append(get_data_point(c, timestamp))
return format_return_data(histo_data, 0, datespan)
def get_active_countries_stats_data(domains, datespan, interval,
datefield='received_on'):
"""
Returns list of timestamps and how many countries were active in the 30
days before the timestamp
"""
histo_data = []
for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
t = timestamp
f = timestamp - relativedelta(days=30)
form_query = (FormES()
.domain(domains)
.terms_aggregation('domain', 'domains')
.submitted(gte=f, lte=t)
.size(0))
active_domains = form_query.run().aggregations.domains.keys
countries = (DomainES()
.in_domains(active_domains)
.terms_aggregation('countries', 'countries')
.size(0))
c = len(countries.run().aggregations.countries.keys)
if c > 0:
histo_data.append(get_data_point(c, timestamp))
return format_return_data(histo_data, 0, datespan)
def _update_calculated_properties():
results = DomainES().filter(
get_domains_to_update_es_filter()
).fields(["name", "_id"]).run().hits
all_stats = all_domain_stats()
active_users_by_domain = {}
for r in results:
dom = r["name"]
domain_obj = Domain.get_by_name(dom)
if not domain_obj:
send_to_elasticsearch("domains", r, delete=True)
continue
try:
props = calced_props(domain_obj, r["_id"], all_stats)
active_users_by_domain[dom] = props['cp_n_active_cc_users']
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
datadog_report_user_stats('commcare.active_mobile_workers.count', active_users_by_domain)
def get_call_center_domains():
result = (DomainES().is_active().filter(
filters.term('call_center_config.enabled', True)).source([
'name', 'default_timezone', 'call_center_config.case_type',
'call_center_config.case_owner_id',
'call_center_config.use_user_location_as_owner',
'call_center_config.use_fixtures'
]).run())
def to_domain_lite(hit):
config = hit.get('call_center_config', {})
case_type = config.get('case_type', None)
case_owner_id = config.get('case_owner_id', None)
use_user_location_as_owner = config.get('use_user_location_as_owner',
None)
if case_type and (case_owner_id or use_user_location_as_owner):
# see CallCenterProperties.config_is_valid()
return DomainLite(
name=hit['name'],
default_timezone=hit['default_timezone'],
cc_case_type=case_type,
use_fixtures=config.get('use_fixtures', True),
form_datasource_enabled=config.get('form_datasource_enabled',
True),
case_datasource_enabled=config.get('case_datasource_enabled',
True),
case_actions_datasource_enabled=config.get(
'case_actions_datasource_enabled', True),
)
return [_f for _f in [to_domain_lite(hit) for hit in result.hits] if _f]
def dehydrate_calculated_properties(self, bundle):
domain = _get_domain(bundle)
try:
es_data = (DomainES()
.in_domains([domain.name])
.size(1)
.run()
.hits[0])
base_properties = {
prop_name: es_data[prop_name]
for prop_name in es_data if prop_name[:3] == 'cp_'
}
try:
audit_record = DomainAuditRecordEntry.objects.get(domain=domain.name)
except DomainAuditRecordEntry.DoesNotExist:
audit_record = None
extra_properties = [
"cp_n_downloads_custom_exports",
"cp_n_viewed_ucr_reports",
"cp_n_viewed_non_ucr_reports",
"cp_n_reports_created",
"cp_n_reports_edited",
"cp_n_saved_scheduled_reports",
"cp_n_click_app_deploy",
"cp_n_form_builder_entered",
"cp_n_saved_app_changes",
]
for prop in extra_properties:
base_properties.update({prop: getattr(audit_record, prop, 0)})
return base_properties
except IndexError:
logging.exception('Problem getting calculated properties for {}'.format(domain.name))
return {}
def update_calculated_properties():
results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
calced_props = {
"_id": r["_id"],
"cp_n_web_users": int(all_stats["web_users"][dom]),
"cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users": int(all_stats["commcare_users"][dom]),
"cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form": total_distinct_users([dom]),
"cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_cases": int(all_stats["cases"][dom]),
"cp_n_forms": int(all_stats["forms"][dom]),
"cp_first_form": CALC_FNS["first_form_submission"](dom, False),
"cp_last_form": CALC_FNS["last_form_submission"](dom, False),
"cp_is_active": CALC_FNS["active"](dom),
"cp_has_app": CALC_FNS["has_app"](dom),
"cp_last_updated": json_format_datetime(datetime.utcnow()),
"cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)),
}
if calced_props['cp_first_form'] == 'No forms':
del calced_props['cp_first_form']
del calced_props['cp_last_form']
send_to_elasticsearch("domains", calced_props)
def get_project_spaces(facets=None):
"""
Returns a list of names of project spaces that satisfy the facets
"""
real_domain_query = (DomainES().fields(["name"]).size(LARGE_ES_NUMBER))
if facets:
real_domain_query = add_params_to_query(real_domain_query, facets)
real_domain_query_results = real_domain_query.run().raw_hits
return [_['fields']['name'] for _ in real_domain_query_results]
def fix_domain_es_docs(apps, schema_editor):
bool_props = ['cp_sms_ever', 'cp_sms_30_d', 'cp_j2me_90_d_bool']
for doc in DomainES().source(bool_props).run().hits:
for prop in bool_props:
doc[prop] = bool(doc.get(prop, False))
send_to_elasticsearch('domains',
doc,
delete=False,
es_merge_update=True)
def get_active_countries_stats_data(domains, datespan, interval, datefield="received_on"):
"""
Returns list of timestamps and how many countries were active in the 30
days before the timestamp
"""
histo_data = []
for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
t = timestamp
f = timestamp - relativedelta(days=30)
form_query = FormES().domain(domains).terms_aggregation("domain", "domains").submitted(gte=f, lte=t).size(0)
active_domains = form_query.run().aggregations.domains.keys
countries = DomainES().in_domains(active_domains).terms_aggregation("countries", "countries").size(0)
c = len(countries.run().aggregations.countries.keys)
if c > 0:
histo_data.append(get_data_point(c, timestamp))
return format_return_data(histo_data, 0, datespan)
def db_comparisons(request):
comparison_config = [{
'description':
'Users (base_doc is "CouchUser")',
'couch_db':
CommCareUser.get_db(),
'view_name':
'users/by_username',
'es_query':
UserES().remove_default_filter('active').remove_default_filter(
'mobile_worker').size(0),
'sql_rows':
User.objects.count(),
}, {
'description': 'Domains (doc_type is "Domain")',
'couch_db': Domain.get_db(),
'view_name': 'domain/by_status',
'es_query': DomainES().size(0),
'sql_rows': None,
}, {
'description':
'Forms (doc_type is "XFormInstance")',
'couch_db':
XFormInstance.get_db(),
'view_name':
'couchforms/by_xmlns',
'es_query':
FormES().remove_default_filter('has_xmlns').remove_default_filter(
'has_user').size(0),
'sql_rows':
FormData.objects.count(),
}, {
'description': 'Cases (doc_type is "CommCareCase")',
'couch_db': CommCareCase.get_db(),
'view_name': 'case/by_owner',
'es_query': CaseES().size(0),
'sql_rows': None,
}]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description':
comp['description'],
'couch_docs':
comp['couch_db'].view(
comp['view_name'],
reduce=True,
).one()['value'],
'es_docs':
comp['es_query'].run().total,
'sql_rows':
comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def _real_incomplete_domains():
incomplete_domains = (
DomainES()
.fields(["name"])
.non_test_domains()
.incomplete_domains()
.run()
.hits
)
return {x['name'] for x in incomplete_domains}
def top_five_projects_by_country(request):
data = {}
if 'country' in request.GET:
country = request.GET.get('country')
projects = (DomainES().is_active().real_domains()
.filter(filters.term('deployment.countries', country))
.sort('cp_n_active_cc_users', True)
.source(['internal.area', 'internal.sub_area', 'cp_n_active_cc_users', 'deployment.countries'])
.size(5).run().hits)
data = {country: projects}
return json_response(data)
def get_project_spaces(facets=None):
"""
Returns a list of names of project spaces that satisfy the facets
"""
real_domain_query = (DomainES().fields(["name"
]).size(DOMAIN_COUNT_UPPER_BOUND))
if facets:
real_domain_query = add_params_to_query(real_domain_query, facets)
domain_names = [hit['name'] for hit in real_domain_query.run().hits]
if 'search' in facets:
return list(set(domain_names) & set(facets['search']))
return domain_names
def get_call_center_domains():
result = (DomainES().is_active().is_snapshot(False).filter(
filters.term('call_center_config.enabled', True)).fields(
['name', 'default_timezone',
'call_center_config.case_type']).run())
def to_domain_lite(hit):
return DomainLite(name=hit['name'],
default_timezone=hit['default_timezone'],
cc_case_type=hit.get('call_center_config.case_type',
''))
return [to_domain_lite(hit) for hit in result.hits]
def dehydrate_calculated_properties(self, bundle):
domain = _get_domain(bundle)
try:
es_data = (DomainES().in_domains([domain.name
]).run().raw_hits[0]['_source'])
return {
raw_hit: es_data[raw_hit]
for raw_hit in es_data if raw_hit[:3] == 'cp_'
}
except IndexError:
logging.exception(
'Problem getting calculated properties for {}'.format(
domain.name))
return {}
def dehydrate_calculated_properties(self, bundle):
domain = _get_domain(bundle)
try:
es_data = (DomainES().in_domains([domain.name
]).size(1).run().hits[0])
return {
prop_name: es_data[prop_name]
for prop_name in es_data if prop_name[:3] == 'cp_'
}
except IndexError:
logging.exception(
'Problem getting calculated properties for {}'.format(
domain.name))
return {}
def get_couch_domains():
"""
Returns a list of dicts of domain properties
"""
return (
DomainES()
.term("use_sql_backend", False)
.size(DOMAIN_COUNT_UPPER_BOUND)
.sort("name")
.values(
"name",
"cp_n_active_cc_users",
"cp_n_forms",
"cp_last_form",
)
)
def top_five_projects_by_country(request):
data = {}
internalMode = request.user.is_superuser
attributes = ['internal.area', 'internal.sub_area', 'cp_n_active_cc_users', 'deployment.countries']
if internalMode:
attributes = ['name', 'internal.organization_name', 'internal.notes'] + attributes
if 'country' in request.GET:
country = request.GET.get('country')
projects = (DomainES().is_active_project().real_domains()
.filter(filters.term('deployment.countries', country))
.sort('cp_n_active_cc_users', True).source(attributes).size(5).run().hits)
data = {country: projects, 'internal': internalMode}
return json_response(data)
def obj_get_list(self, bundle, **kwargs):
if kwargs.get('domain'):
return [self.obj_get(bundle, **kwargs)]
else:
filters = {}
if hasattr(bundle.request, 'GET'):
filters = bundle.request.GET
params = {}
if 'last_modified__lte' in filters:
params['lte'] = force_to_datetime(filters['last_modified__lte'])
if 'last_modified__gte' in filters:
params['gte'] = force_to_datetime(filters['last_modified__gte'])
return DomainQuerySetAdapter(DomainES().last_modified(**params).sort('last_modified'))
def get_countries_stats_data(domains,
datespan,
interval,
datefield='created_on'):
"""
Returns list of timestamps and how many countries have been created before
each interval
"""
histo_data = []
for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
countries = (DomainES().in_domains(domains).created(
lte=timestamp).terms_aggregation('countries', 'countries').size(0))
c = len(countries.run().aggregations.countries.keys)
if c > 0:
histo_data.append(get_data_point(c, timestamp))
return format_return_data(histo_data, 0, datespan)
def update_calculated_properties():
results = DomainES().fields(["name", "_id", "cp_last_updated"]).scroll()
all_stats = all_domain_stats()
for r in results:
dom = r["name"]
try:
last_form_submission = CALC_FNS["last_form_submission"](dom, False)
if _skip_updating_domain_stats(r.get("cp_last_updated"), last_form_submission):
continue
props = calced_props(dom, r["_id"], all_stats)
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def db_comparisons(request):
def _simple_view_couch_query(db, view_name):
return db.view(view_name, reduce=True).one()['value']
comparison_config = [
{
'description': 'Users (base_doc is "CouchUser")',
'couch_docs': _simple_view_couch_query(CommCareUser.get_db(), 'users/by_username'),
'es_query': UserES().remove_default_filter('active').size(0),
'sql_rows': User.objects.count(),
},
{
'description': 'Domains (doc_type is "Domain")',
'couch_docs': _simple_view_couch_query(Domain.get_db(), 'domain/by_status'),
'es_query': DomainES().size(0),
'sql_rows': None,
},
{
'description': 'Forms (doc_type is "XFormInstance")',
'couch_docs': get_number_of_forms_all_domains_in_couch(),
'es_query': FormES().remove_default_filter('has_xmlns')
.remove_default_filter('has_user')
.size(0),
'sql_rows': FormData.objects.exclude(domain__isnull=True).count(),
},
{
'description': 'Cases (doc_type is "CommCareCase")',
'couch_docs': get_total_case_count(),
'es_query': CaseES().size(0),
'sql_rows': CaseData.objects.exclude(domain__isnull=True).count(),
}
]
comparisons = []
for comp in comparison_config:
comparisons.append({
'description': comp['description'],
'couch_docs': comp['couch_docs'],
'es_docs': comp['es_query'].run().total,
'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a',
})
return json_response(comparisons)
def update_calculated_properties():
results = DomainES().fields(["name", "_id"]).run().hits
all_stats = _all_domain_stats()
for r in results:
dom = r["name"]
try:
calced_props = {
"_id":
r["_id"],
"cp_n_web_users":
int(all_stats["web_users"].get(dom, 0)),
"cp_n_active_cc_users":
int(CALC_FNS["mobile_users"](dom)),
"cp_n_cc_users":
int(all_stats["commcare_users"].get(dom, 0)),
"cp_n_active_cases":
int(CALC_FNS["cases_in_last"](dom, 120)),
"cp_n_users_submitted_form":
total_distinct_users([dom]),
"cp_n_inactive_cases":
int(CALC_FNS["inactive_cases_in_last"](dom, 120)),
"cp_n_30_day_cases":
int(CALC_FNS["cases_in_last"](dom, 30)),
"cp_n_60_day_cases":
int(CALC_FNS["cases_in_last"](dom, 60)),
"cp_n_90_day_cases":
int(CALC_FNS["cases_in_last"](dom, 90)),
"cp_n_cases":
int(all_stats["cases"].get(dom, 0)),
"cp_n_forms":
int(all_stats["forms"].get(dom, 0)),
"cp_n_forms_30_d":
int(CALC_FNS["forms_in_last"](dom, 30)),
"cp_n_forms_60_d":
int(CALC_FNS["forms_in_last"](dom, 60)),
"cp_n_forms_90_d":
int(CALC_FNS["forms_in_last"](dom, 90)),
"cp_first_form":
CALC_FNS["first_form_submission"](dom, False),
"cp_last_form":
CALC_FNS["last_form_submission"](dom, False),
"cp_is_active":
CALC_FNS["active"](dom),
"cp_has_app":
CALC_FNS["has_app"](dom),
"cp_last_updated":
json_format_datetime(datetime.utcnow()),
"cp_n_in_sms":
int(CALC_FNS["sms"](dom, "I")),
"cp_n_out_sms":
int(CALC_FNS["sms"](dom, "O")),
"cp_n_sms_ever":
int(CALC_FNS["sms_in_last"](dom)),
"cp_n_sms_30_d":
int(CALC_FNS["sms_in_last"](dom, 30)),
"cp_n_sms_60_d":
int(CALC_FNS["sms_in_last"](dom, 60)),
"cp_n_sms_90_d":
int(CALC_FNS["sms_in_last"](dom, 90)),
"cp_sms_ever":
int(CALC_FNS["sms_in_last_bool"](dom)),
"cp_sms_30_d":
int(CALC_FNS["sms_in_last_bool"](dom, 30)),
"cp_n_sms_in_30_d":
int(CALC_FNS["sms_in_in_last"](dom, 30)),
"cp_n_sms_in_60_d":
int(CALC_FNS["sms_in_in_last"](dom, 60)),
"cp_n_sms_in_90_d":
int(CALC_FNS["sms_in_in_last"](dom, 90)),
"cp_n_sms_out_30_d":
int(CALC_FNS["sms_out_in_last"](dom, 30)),
"cp_n_sms_out_60_d":
int(CALC_FNS["sms_out_in_last"](dom, 60)),
"cp_n_sms_out_90_d":
int(CALC_FNS["sms_out_in_last"](dom, 90)),
"cp_n_j2me_30_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 30)),
"cp_n_j2me_60_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 60)),
"cp_n_j2me_90_d":
int(CALC_FNS["j2me_forms_in_last"](dom, 90)),
"cp_j2me_90_d_bool":
int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)),
"cp_300th_form":
CALC_FNS["300th_form_submission"](dom)
}
if calced_props['cp_first_form'] is None:
del calced_props['cp_first_form']
if calced_props['cp_last_form'] is None:
del calced_props['cp_last_form']
if calced_props['cp_300th_form'] is None:
del calced_props['cp_300th_form']
send_to_elasticsearch("domains", calced_props)
except Exception, e:
notify_exception(
None,
message='Domain {} failed on stats calculations with {}'.
format(dom, e))
