def _domains_over_x_forms(num_forms=200, domains=None):
form_domains = FormES().domain_facet().size(0)
if domains:
form_domains = form_domains.domain(domains)
form_domains = form_domains.run().facet('domain', 'terms')
return {x['term'] for x in form_domains if x['count'] > num_forms}
def get_domain_device_breakdown_es(domain_name, monthspan):
query = FormES(es_instance_alias=ES_EXPORT_INSTANCE).domain(domain_name).submitted(
gte=monthspan.startdate,
lt=monthspan.computed_enddate,
).aggregation(TermsAggregation('device_id', 'form.meta.deviceID')).size(0)
return query.run().aggregations.device_id.counts_by_bucket()
def _domains_over_x_forms(num_forms=200, domains=None):
form_domains = FormES().domain_aggregation().size(0)
if domains:
form_domains = form_domains.domain(domains)
form_domains = form_domains.run().aggregations.domain.buckets_list
return {x.key for x in form_domains if x.doc_count > num_forms}
def _domains_over_x_forms(num_forms=200, domains=None):
form_domains = FormES().domain_facet().size(0)
if domains:
form_domains = form_domains.domain(domains)
form_domains = form_domains.run().facet('domain', 'terms')
return {x['term'] for x in form_domains if x['count'] > num_forms}
def get_sms_only_domain_stats_data(domains, datespan, interval, datefield="date_created"):
"""
Returns domains that have only used SMS and not forms.
Returned based on date domain is created
"""
histo_data = []
sms = SMSES().domain(domains).terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND).size(0)
forms = FormES().domain(domains).terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND).size(0)
sms_domains = {x["term"] for x in sms.run().facet("domains", "terms")}
form_domains = {x["term"] for x in forms.run().facet("domains", "terms")}
sms_only_domains = sms_domains - form_domains
domains_after_date = (
DomainES()
.in_domains(sms_only_domains)
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.size(0)
)
histo_data = domains_after_date.run().facet("date", "entries")
domains_before_date = DomainES().in_domains(sms_only_domains).created(lt=datespan.startdate).size(0)
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def _domains_over_x_forms(num_forms=200, domains=None):
form_domains = FormES().domain_aggregation().size(0)
if domains:
form_domains = form_domains.domain(domains)
form_domains = form_domains.run().aggregations.domain.buckets_list
return {x.key for x in form_domains if x.doc_count > num_forms}
def get_domain_device_breakdown_es(domain_name, monthspan):
query = FormES(for_export=True).domain(domain_name).submitted(
gte=monthspan.startdate,
lt=monthspan.computed_enddate,
).aggregation(TermsAggregation('device_id', 'form.meta.deviceID')).size(0)
return query.run().aggregations.device_id.counts_by_bucket()
def _domains_over_x_forms(num_forms=200, domains=None):
form_domains = FormES().domain_facet().size(0)
if domains:
form_domains = form_domains.domain(domains)
form_domains = form_domains.run().facet("domain", "terms")
return {x["term"] for x in form_domains if x["count"] > num_forms}
def get_domain_device_breakdown_es(domain_name, monthspan):
query = FormES().domain(domain_name).submitted(
gte=monthspan.startdate,
lt=monthspan.computed_enddate,
).aggregation(TermsAggregation('device_id', 'form.meta.deviceID')).size(0)
return query.run().aggregations.device_id.counts_by_bucket()
def get_sms_only_domain_stats_data(domains, datespan, interval, datefield="date_created"):
"""
Returns domains that have only used SMS and not forms.
Returned based on date domain is created
"""
histo_data = []
sms = SMSES().domain(domains).terms_aggregation("domain", "domains").size(0)
forms = FormES().domain(domains).terms_aggregation("domain", "domains").size(0)
sms_domains = set(sms.run().aggregations.domains.keys)
form_domains = set(forms.run().aggregations.domains.keys)
sms_only_domains = sms_domains - form_domains
domains_after_date = (
DomainES()
.in_domains(sms_only_domains)
.created(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.size(0)
)
histo_data = domains_after_date.run().aggregations.date.as_facet_result()
domains_before_date = DomainES().in_domains(sms_only_domains).created(lt=datespan.startdate).size(0)
domains_before_date = domains_before_date.run().total
return format_return_data(histo_data, domains_before_date, datespan)
def test_not_filter_edge_case(self):
self._setup_data()
query = FormES().remove_default_filters().filter(
filters.NOT(filters.OR(
filters.term('domain', 'd'),
filters.term('app_id', 'a')
))
)
self.assertEqual(query.run().doc_ids, ['doc3'])
def total_distinct_users(domain):
"""
Get total number of users who've ever submitted a form in a domain.
"""
query = FormES().domain(domain).user_aggregation()
terms = {
user_id for user_id in query.run().aggregations.user.keys
if user_id not in WEIRD_USER_IDS
}
user_ids = terms.intersection(set(CouchUser.ids_by_domain(domain)))
return len(user_ids)
def total_distinct_users(domain):
"""
Get total number of users who've ever submitted a form in a domain.
"""
query = FormES().domain(domain).user_aggregation()
terms = {
user_id for user_id in query.run().aggregations.user.keys
if user_id not in WEIRD_USER_IDS
}
user_ids = terms.intersection(set(CouchUser.ids_by_domain(domain)))
return len(user_ids)
def commtrack_form_submissions(domains, datespan, interval, datefield="received_on"):
mobile_workers = [a["_id"] for a in UserES().fields([]).mobile_users().show_inactive().run().raw_hits]
forms_after_date = (
FormES()
.domain(domains)
.submitted(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.user_id(mobile_workers)
.size(0)
)
histo_data = forms_after_date.run().facet("date", "entries")
forms_before_date = FormES().domain(domains).submitted(lt=datespan.startdate).user_id(mobile_workers).size(0)
forms_before_date = forms_before_date.run().total
return format_return_data(histo_data, forms_before_date, datespan)
def commtrack_form_submissions(domains, datespan, interval, datefield="received_on"):
mobile_workers = get_mobile_users(domains)
forms_after_date = (
FormES()
.domain(domains)
.submitted(gte=datespan.startdate, lte=datespan.enddate)
.date_histogram("date", datefield, interval)
.user_id(mobile_workers)
.size(0)
)
histo_data = forms_after_date.run().aggregations.date.as_facet_result()
forms_before_date = FormES().domain(domains).submitted(lt=datespan.startdate).user_id(mobile_workers).size(0)
forms_before_date = forms_before_date.run().total
return format_return_data(histo_data, forms_before_date, datespan)
def get_active_countries_stats_data(domains, datespan, interval, datefield="received_on"):
"""
Returns list of timestamps and how many countries were active in the 30
days before the timestamp
"""
histo_data = []
for timestamp in daterange(interval, datespan.startdate, datespan.enddate):
t = timestamp
f = timestamp - relativedelta(days=30)
form_query = FormES().domain(domains).terms_aggregation("domain", "domains").submitted(gte=f, lte=t).size(0)
active_domains = form_query.run().aggregations.domains.keys
countries = DomainES().in_domains(active_domains).terms_aggregation("countries", "countries").size(0)
c = len(countries.run().aggregations.countries.keys)
if c > 0:
histo_data.append(get_data_point(c, timestamp))
return format_return_data(histo_data, 0, datespan)