def get_active_countries_stats_data(domains, datespan, interval, datefield="received_on"): """ Returns list of timestamps and how many countries were active in the 30 days before the timestamp """ histo_data = [] for timestamp in daterange(interval, datespan.startdate, datespan.enddate): t = timestamp f = timestamp - relativedelta(days=30) form_query = ( FormES() .domain(domains) .terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND) .submitted(gte=f, lte=t) .size(0) ) domains = form_query.run().facet("domains", "terms") domains = [x["term"] for x in domains] countries = DomainES().in_domains(domains).terms_facet("countries", "countries", size=COUNTRY_COUNT_UPPER_BOUND) c = len(countries.run().facet("countries", "terms")) if c > 0: histo_data.append(get_data_point(c, timestamp)) return format_return_data(histo_data, 0, datespan)
def handle(self, *args, **options): # Find all apps using vellum case management app_query = AppES().is_build(False).term( 'vellum_case_management', True).source(['domain', '_id']) # Find all domains created after vellum case management was released, # since their apps were never in the old world and therefore never migrated epoch = datetime(year=2016, month=7, day=15) domain_query = DomainES().date_range('date_created', gt=epoch).source('name') new_domains = {d['name']: 1 for d in domain_query.run().hits} hits = app_query.run().hits for hit in hits: if hit['domain'] not in new_domains: try: app = get_app(hit['domain'], hit['_id']) modules = [ m for m in app.modules if m.module_type == 'basic' ] for module in modules: forms = [ f for f in module.forms if f.doc_type == 'Form' ] for form in forms: if form.requires != 'case' and 'load' in form.case_references: if form.case_references['load']: logger.info( '{} is suspicious: /a/{}/apps/view/{}/modules-{}/forms-{}' .format(form.unique_id, app.domain, app.id, module.id, form.id)) except Http404: pass logger.info('done with cmitfb_identify_broken_forms')
def get_commconnect_domain_stats_data(domains, datespan, interval, datefield="date_created", additional_params_es={}): """ Returns domains that have used SMS. Returned based on date domain is created """ sms = SMSES().domain(domains).terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND).size(0) if additional_params_es: sms = add_params_to_query(sms, additional_params_es) sms_domains = {x["term"] for x in sms.run().facet("domains", "terms")} domains_after_date = ( DomainES() .in_domains(sms_domains) .created(gte=datespan.startdate, lte=datespan.enddate) .date_histogram("date", datefield, interval) .size(0) ) histo_data = domains_after_date.run().facet("date", "entries") domains_before_date = DomainES().in_domains(sms_domains).created(lt=datespan.startdate).size(0) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_commconnect_domain_stats_data(domains, datespan, interval, datefield='date_created', additional_params_es={}): """ Returns domains that have used SMS. Returned based on date domain is created """ sms = (SMSES().domain(domains).terms_facet('domain', 'domains', size=LARGE_ES_NUMBER).size(0)) if additional_params_es: sms = add_params_to_query(sms, additional_params_es) sms_domains = {x['term'] for x in sms.run().facet('domains', 'terms')} domains_after_date = (DomainES().in_domains(sms_domains).created( gte=datespan.startdate, lte=datespan.enddate).date_histogram('date', datefield, interval).size(0)) histo_data = domains_after_date.run().facet('date', 'entries') domains_before_date = (DomainES().in_domains(sms_domains).created( lt=datespan.startdate).size(0)) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_sms_only_domain_stats_data(domains, datespan, interval, datefield='date_created'): """ Returns domains that have only used SMS and not forms. Returned based on date domain is created """ histo_data = [] sms = (SMSES().domain(domains).terms_facet('domain', 'domains', size=LARGE_ES_NUMBER).size(0)) forms = (FormES().domain(domains).terms_facet( 'domain', 'domains', size=LARGE_ES_NUMBER).size(0)) sms_domains = {x['term'] for x in sms.run().facet('domains', 'terms')} form_domains = {x['term'] for x in forms.run().facet('domains', 'terms')} sms_only_domains = sms_domains - form_domains domains_after_date = (DomainES().in_domains(sms_only_domains).created( gte=datespan.startdate, lte=datespan.enddate).date_histogram('date', datefield, interval).size(0)) histo_data = domains_after_date.run().facet('date', 'entries') domains_before_date = (DomainES().in_domains(sms_only_domains).created( lt=datespan.startdate).size(0)) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_sms_only_domain_stats_data(domains, datespan, interval, datefield="date_created"): """ Returns domains that have only used SMS and not forms. Returned based on date domain is created """ histo_data = [] sms = SMSES().domain(domains).terms_aggregation("domain", "domains").size(0) forms = FormES().domain(domains).terms_aggregation("domain", "domains").size(0) sms_domains = set(sms.run().aggregations.domains.keys) form_domains = set(forms.run().aggregations.domains.keys) sms_only_domains = sms_domains - form_domains domains_after_date = ( DomainES() .in_domains(sms_only_domains) .created(gte=datespan.startdate, lte=datespan.enddate) .date_histogram("date", datefield, interval) .size(0) ) histo_data = domains_after_date.run().aggregations.date.as_facet_result() domains_before_date = DomainES().in_domains(sms_only_domains).created(lt=datespan.startdate).size(0) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_commconnect_domain_stats_data(domains, datespan, interval, datefield="date_created", additional_params_es={}): """ Returns domains that have used SMS. Returned based on date domain is created """ sms = SMSES().domain(domains).terms_aggregation("domain", "domains").size(0) if additional_params_es: sms = add_params_to_query(sms, additional_params_es) sms_domains = set(sms.run().aggregations.domains.keys) domains_after_date = ( DomainES() .in_domains(sms_domains) .created(gte=datespan.startdate, lte=datespan.enddate) .date_histogram("date", datefield, interval) .size(0) ) histo_data = domains_after_date.run().aggregations.date.as_facet_result() domains_before_date = DomainES().in_domains(sms_domains).created(lt=datespan.startdate).size(0) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_sms_only_domain_stats_data(domains, datespan, interval, datefield='date_created'): """ Returns domains that have only used SMS and not forms. Returned based on date domain is created """ histo_data = [] sms = (SMSES().domain(domains).terms_aggregation('domain', 'domains').size(0)) forms = (FormES().domain(domains).terms_aggregation('domain', 'domains').size(0)) sms_domains = set(sms.run().aggregations.domains.keys) form_domains = set(forms.run().aggregations.domains.keys) sms_only_domains = sms_domains - form_domains domains_after_date = (DomainES().in_domains(sms_only_domains).created( gte=datespan.startdate, lte=datespan.enddate).date_histogram('date', datefield, interval).size(0)) histo_data = domains_after_date.run().aggregations.date.as_facet_result() domains_before_date = (DomainES().in_domains(sms_only_domains).created( lt=datespan.startdate).size(0)) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_commconnect_domain_stats_data(domains, datespan, interval, datefield='date_created', additional_params_es={}): """ Returns domains that have used SMS. Returned based on date domain is created """ sms = (SMSES().domain(domains).terms_aggregation('domain', 'domains').size(0)) if additional_params_es: sms = add_params_to_query(sms, additional_params_es) sms_domains = set(sms.run().aggregations.domains.keys) domains_after_date = (DomainES().in_domains(sms_domains).created( gte=datespan.startdate, lte=datespan.enddate).date_histogram('date', datefield, interval).size(0)) histo_data = domains_after_date.run().aggregations.date.as_facet_result() domains_before_date = (DomainES().in_domains(sms_domains).created( lt=datespan.startdate).size(0)) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_sms_only_domain_stats_data(domains, datespan, interval, datefield="date_created"): """ Returns domains that have only used SMS and not forms. Returned based on date domain is created """ histo_data = [] sms = SMSES().domain(domains).terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND).size(0) forms = FormES().domain(domains).terms_facet("domain", "domains", size=DOMAIN_COUNT_UPPER_BOUND).size(0) sms_domains = {x["term"] for x in sms.run().facet("domains", "terms")} form_domains = {x["term"] for x in forms.run().facet("domains", "terms")} sms_only_domains = sms_domains - form_domains domains_after_date = ( DomainES() .in_domains(sms_only_domains) .created(gte=datespan.startdate, lte=datespan.enddate) .date_histogram("date", datefield, interval) .size(0) ) histo_data = domains_after_date.run().facet("date", "entries") domains_before_date = DomainES().in_domains(sms_only_domains).created(lt=datespan.startdate).size(0) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_project_spaces(facets=None): """ Returns a list of names of project spaces that satisfy the facets """ real_domain_query = DomainES().fields(["name"]).size(DOMAIN_COUNT_UPPER_BOUND) if facets: real_domain_query = add_params_to_query(real_domain_query, facets) domain_names = [hit["name"] for hit in real_domain_query.run().hits] if "search" in facets: return list(set(domain_names) & set(facets["search"])) return domain_names
def get_domain_stats_data(domains, datespan, interval, datefield="date_created"): domains_after_date = ( DomainES() .in_domains(domains) .created(gte=datespan.startdate, lte=datespan.enddate) .date_histogram("date", datefield, interval) .size(0) ) histo_data = domains_after_date.run().facet("date", "entries") domains_before_date = DomainES().in_domains(domains).created(lt=datespan.startdate).size(0) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def dehydrate_calculated_properties(self, bundle): calc_prop_prefix = 'cp_' domain_obj = _get_domain(bundle) try: es_data = (DomainES() .in_domains([domain_obj.name]) .size(1) .run() .hits[0]) base_properties = { prop_name: es_data[prop_name] for prop_name in es_data if prop_name.startswith(calc_prop_prefix) } try: audit_record = DomainAuditRecordEntry.objects.get(domain=domain_obj.name) except DomainAuditRecordEntry.DoesNotExist: audit_record = None extra_properties = { field.name: getattr(audit_record, field.name, 0) for field in DomainAuditRecordEntry._meta.fields if field.name.startswith(calc_prop_prefix) } base_properties.update(extra_properties) return base_properties except IndexError: logging.exception('Problem getting calculated properties for {}'.format(domain_obj.name)) return {}
def update_calculated_properties(): results = DomainES().filter(get_domains_to_update_es_filter()).fields( ["name", "_id", "cp_last_updated"]).scroll() all_stats = all_domain_stats() for r in results: dom = r["name"] domain_obj = Domain.get_by_name(dom) if not domain_obj: send_to_elasticsearch("domains", r, delete=True) continue try: props = calced_props(domain_obj, r["_id"], all_stats) if props['cp_first_form'] is None: del props['cp_first_form'] if props['cp_last_form'] is None: del props['cp_last_form'] if props['cp_300th_form'] is None: del props['cp_300th_form'] send_to_elasticsearch("domains", props, es_merge_update=True) except Exception as e: notify_exception( None, message='Domain {} failed on stats calculations with {}'. format(dom, e))
def get_domain_stats_data(domains, datespan, interval, datefield="date_created"): domains_after_date = (DomainES().in_domains(domains).created( gte=datespan.startdate, lte=datespan.enddate).date_histogram('date', datefield, interval).size(0)) histo_data = domains_after_date.run().aggregations.date.as_facet_result() domains_before_date = (DomainES().in_domains(domains).created( lt=datespan.startdate).size(0)) domains_before_date = domains_before_date.run().total return format_return_data(histo_data, domains_before_date, datespan)
def get_active_countries_stats_data(domains, datespan, interval, datefield='received_on'): """ Returns list of timestamps and how many countries were active in the 30 days before the timestamp """ histo_data = [] for timestamp in daterange(interval, datespan.startdate, datespan.enddate): t = timestamp f = timestamp - relativedelta(days=30) form_query = (FormES().domain(domains).terms_facet( 'domain', 'domains', size=LARGE_ES_NUMBER).submitted(gte=f, lte=t).size(0)) domains = form_query.run().facet('domains', "terms") domains = [x['term'] for x in domains] countries = (DomainES().in_domains(domains).terms_facet( 'countries', 'countries', size=LARGE_ES_NUMBER)) c = len(countries.run().facet('countries', 'terms')) if c > 0: histo_data.append(get_data_point(c, timestamp)) return format_return_data(histo_data, 0, datespan)
def get_active_countries_stats_data(domains, datespan, interval, datefield='received_on'): """ Returns list of timestamps and how many countries were active in the 30 days before the timestamp """ histo_data = [] for timestamp in daterange(interval, datespan.startdate, datespan.enddate): t = timestamp f = timestamp - relativedelta(days=30) form_query = (FormES() .domain(domains) .terms_aggregation('domain', 'domains') .submitted(gte=f, lte=t) .size(0)) active_domains = form_query.run().aggregations.domains.keys countries = (DomainES() .in_domains(active_domains) .terms_aggregation('countries', 'countries') .size(0)) c = len(countries.run().aggregations.countries.keys) if c > 0: histo_data.append(get_data_point(c, timestamp)) return format_return_data(histo_data, 0, datespan)
def _update_calculated_properties(): results = DomainES().filter( get_domains_to_update_es_filter() ).fields(["name", "_id"]).run().hits all_stats = all_domain_stats() active_users_by_domain = {} for r in results: dom = r["name"] domain_obj = Domain.get_by_name(dom) if not domain_obj: send_to_elasticsearch("domains", r, delete=True) continue try: props = calced_props(domain_obj, r["_id"], all_stats) active_users_by_domain[dom] = props['cp_n_active_cc_users'] if props['cp_first_form'] is None: del props['cp_first_form'] if props['cp_last_form'] is None: del props['cp_last_form'] if props['cp_300th_form'] is None: del props['cp_300th_form'] send_to_elasticsearch("domains", props, es_merge_update=True) except Exception as e: notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e)) datadog_report_user_stats('commcare.active_mobile_workers.count', active_users_by_domain)
def get_call_center_domains(): result = (DomainES().is_active().filter( filters.term('call_center_config.enabled', True)).source([ 'name', 'default_timezone', 'call_center_config.case_type', 'call_center_config.case_owner_id', 'call_center_config.use_user_location_as_owner', 'call_center_config.use_fixtures' ]).run()) def to_domain_lite(hit): config = hit.get('call_center_config', {}) case_type = config.get('case_type', None) case_owner_id = config.get('case_owner_id', None) use_user_location_as_owner = config.get('use_user_location_as_owner', None) if case_type and (case_owner_id or use_user_location_as_owner): # see CallCenterProperties.config_is_valid() return DomainLite( name=hit['name'], default_timezone=hit['default_timezone'], cc_case_type=case_type, use_fixtures=config.get('use_fixtures', True), form_datasource_enabled=config.get('form_datasource_enabled', True), case_datasource_enabled=config.get('case_datasource_enabled', True), case_actions_datasource_enabled=config.get( 'case_actions_datasource_enabled', True), ) return [_f for _f in [to_domain_lite(hit) for hit in result.hits] if _f]
def dehydrate_calculated_properties(self, bundle): domain = _get_domain(bundle) try: es_data = (DomainES() .in_domains([domain.name]) .size(1) .run() .hits[0]) base_properties = { prop_name: es_data[prop_name] for prop_name in es_data if prop_name[:3] == 'cp_' } try: audit_record = DomainAuditRecordEntry.objects.get(domain=domain.name) except DomainAuditRecordEntry.DoesNotExist: audit_record = None extra_properties = [ "cp_n_downloads_custom_exports", "cp_n_viewed_ucr_reports", "cp_n_viewed_non_ucr_reports", "cp_n_reports_created", "cp_n_reports_edited", "cp_n_saved_scheduled_reports", "cp_n_click_app_deploy", "cp_n_form_builder_entered", "cp_n_saved_app_changes", ] for prop in extra_properties: base_properties.update({prop: getattr(audit_record, prop, 0)}) return base_properties except IndexError: logging.exception('Problem getting calculated properties for {}'.format(domain.name)) return {}
def update_calculated_properties(): results = DomainES().is_snapshot(False).fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"][dom]), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"][dom]), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_cases": int(all_stats["cases"][dom]), "cp_n_forms": int(all_stats["forms"][dom]), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), } if calced_props['cp_first_form'] == 'No forms': del calced_props['cp_first_form'] del calced_props['cp_last_form'] send_to_elasticsearch("domains", calced_props)
def get_project_spaces(facets=None): """ Returns a list of names of project spaces that satisfy the facets """ real_domain_query = (DomainES().fields(["name"]).size(LARGE_ES_NUMBER)) if facets: real_domain_query = add_params_to_query(real_domain_query, facets) real_domain_query_results = real_domain_query.run().raw_hits return [_['fields']['name'] for _ in real_domain_query_results]
def fix_domain_es_docs(apps, schema_editor): bool_props = ['cp_sms_ever', 'cp_sms_30_d', 'cp_j2me_90_d_bool'] for doc in DomainES().source(bool_props).run().hits: for prop in bool_props: doc[prop] = bool(doc.get(prop, False)) send_to_elasticsearch('domains', doc, delete=False, es_merge_update=True)
def get_active_countries_stats_data(domains, datespan, interval, datefield="received_on"): """ Returns list of timestamps and how many countries were active in the 30 days before the timestamp """ histo_data = [] for timestamp in daterange(interval, datespan.startdate, datespan.enddate): t = timestamp f = timestamp - relativedelta(days=30) form_query = FormES().domain(domains).terms_aggregation("domain", "domains").submitted(gte=f, lte=t).size(0) active_domains = form_query.run().aggregations.domains.keys countries = DomainES().in_domains(active_domains).terms_aggregation("countries", "countries").size(0) c = len(countries.run().aggregations.countries.keys) if c > 0: histo_data.append(get_data_point(c, timestamp)) return format_return_data(histo_data, 0, datespan)
def db_comparisons(request): comparison_config = [{ 'description': 'Users (base_doc is "CouchUser")', 'couch_db': CommCareUser.get_db(), 'view_name': 'users/by_username', 'es_query': UserES().remove_default_filter('active').remove_default_filter( 'mobile_worker').size(0), 'sql_rows': User.objects.count(), }, { 'description': 'Domains (doc_type is "Domain")', 'couch_db': Domain.get_db(), 'view_name': 'domain/by_status', 'es_query': DomainES().size(0), 'sql_rows': None, }, { 'description': 'Forms (doc_type is "XFormInstance")', 'couch_db': XFormInstance.get_db(), 'view_name': 'couchforms/by_xmlns', 'es_query': FormES().remove_default_filter('has_xmlns').remove_default_filter( 'has_user').size(0), 'sql_rows': FormData.objects.count(), }, { 'description': 'Cases (doc_type is "CommCareCase")', 'couch_db': CommCareCase.get_db(), 'view_name': 'case/by_owner', 'es_query': CaseES().size(0), 'sql_rows': None, }] comparisons = [] for comp in comparison_config: comparisons.append({ 'description': comp['description'], 'couch_docs': comp['couch_db'].view( comp['view_name'], reduce=True, ).one()['value'], 'es_docs': comp['es_query'].run().total, 'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a', }) return json_response(comparisons)
def _real_incomplete_domains(): incomplete_domains = ( DomainES() .fields(["name"]) .non_test_domains() .incomplete_domains() .run() .hits ) return {x['name'] for x in incomplete_domains}
def top_five_projects_by_country(request): data = {} if 'country' in request.GET: country = request.GET.get('country') projects = (DomainES().is_active().real_domains() .filter(filters.term('deployment.countries', country)) .sort('cp_n_active_cc_users', True) .source(['internal.area', 'internal.sub_area', 'cp_n_active_cc_users', 'deployment.countries']) .size(5).run().hits) data = {country: projects} return json_response(data)
def get_project_spaces(facets=None): """ Returns a list of names of project spaces that satisfy the facets """ real_domain_query = (DomainES().fields(["name" ]).size(DOMAIN_COUNT_UPPER_BOUND)) if facets: real_domain_query = add_params_to_query(real_domain_query, facets) domain_names = [hit['name'] for hit in real_domain_query.run().hits] if 'search' in facets: return list(set(domain_names) & set(facets['search'])) return domain_names
def get_call_center_domains(): result = (DomainES().is_active().is_snapshot(False).filter( filters.term('call_center_config.enabled', True)).fields( ['name', 'default_timezone', 'call_center_config.case_type']).run()) def to_domain_lite(hit): return DomainLite(name=hit['name'], default_timezone=hit['default_timezone'], cc_case_type=hit.get('call_center_config.case_type', '')) return [to_domain_lite(hit) for hit in result.hits]
def dehydrate_calculated_properties(self, bundle): domain = _get_domain(bundle) try: es_data = (DomainES().in_domains([domain.name ]).run().raw_hits[0]['_source']) return { raw_hit: es_data[raw_hit] for raw_hit in es_data if raw_hit[:3] == 'cp_' } except IndexError: logging.exception( 'Problem getting calculated properties for {}'.format( domain.name)) return {}
def dehydrate_calculated_properties(self, bundle): domain = _get_domain(bundle) try: es_data = (DomainES().in_domains([domain.name ]).size(1).run().hits[0]) return { prop_name: es_data[prop_name] for prop_name in es_data if prop_name[:3] == 'cp_' } except IndexError: logging.exception( 'Problem getting calculated properties for {}'.format( domain.name)) return {}
def get_couch_domains(): """ Returns a list of dicts of domain properties """ return ( DomainES() .term("use_sql_backend", False) .size(DOMAIN_COUNT_UPPER_BOUND) .sort("name") .values( "name", "cp_n_active_cc_users", "cp_n_forms", "cp_last_form", ) )
def top_five_projects_by_country(request): data = {} internalMode = request.user.is_superuser attributes = ['internal.area', 'internal.sub_area', 'cp_n_active_cc_users', 'deployment.countries'] if internalMode: attributes = ['name', 'internal.organization_name', 'internal.notes'] + attributes if 'country' in request.GET: country = request.GET.get('country') projects = (DomainES().is_active_project().real_domains() .filter(filters.term('deployment.countries', country)) .sort('cp_n_active_cc_users', True).source(attributes).size(5).run().hits) data = {country: projects, 'internal': internalMode} return json_response(data)
def obj_get_list(self, bundle, **kwargs): if kwargs.get('domain'): return [self.obj_get(bundle, **kwargs)] else: filters = {} if hasattr(bundle.request, 'GET'): filters = bundle.request.GET params = {} if 'last_modified__lte' in filters: params['lte'] = force_to_datetime(filters['last_modified__lte']) if 'last_modified__gte' in filters: params['gte'] = force_to_datetime(filters['last_modified__gte']) return DomainQuerySetAdapter(DomainES().last_modified(**params).sort('last_modified'))
def get_countries_stats_data(domains, datespan, interval, datefield='created_on'): """ Returns list of timestamps and how many countries have been created before each interval """ histo_data = [] for timestamp in daterange(interval, datespan.startdate, datespan.enddate): countries = (DomainES().in_domains(domains).created( lte=timestamp).terms_aggregation('countries', 'countries').size(0)) c = len(countries.run().aggregations.countries.keys) if c > 0: histo_data.append(get_data_point(c, timestamp)) return format_return_data(histo_data, 0, datespan)
def update_calculated_properties(): results = DomainES().fields(["name", "_id", "cp_last_updated"]).scroll() all_stats = all_domain_stats() for r in results: dom = r["name"] try: last_form_submission = CALC_FNS["last_form_submission"](dom, False) if _skip_updating_domain_stats(r.get("cp_last_updated"), last_form_submission): continue props = calced_props(dom, r["_id"], all_stats) if props['cp_first_form'] is None: del props['cp_first_form'] if props['cp_last_form'] is None: del props['cp_last_form'] if props['cp_300th_form'] is None: del props['cp_300th_form'] send_to_elasticsearch("domains", props, es_merge_update=True) except Exception as e: notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
def db_comparisons(request): def _simple_view_couch_query(db, view_name): return db.view(view_name, reduce=True).one()['value'] comparison_config = [ { 'description': 'Users (base_doc is "CouchUser")', 'couch_docs': _simple_view_couch_query(CommCareUser.get_db(), 'users/by_username'), 'es_query': UserES().remove_default_filter('active').size(0), 'sql_rows': User.objects.count(), }, { 'description': 'Domains (doc_type is "Domain")', 'couch_docs': _simple_view_couch_query(Domain.get_db(), 'domain/by_status'), 'es_query': DomainES().size(0), 'sql_rows': None, }, { 'description': 'Forms (doc_type is "XFormInstance")', 'couch_docs': get_number_of_forms_all_domains_in_couch(), 'es_query': FormES().remove_default_filter('has_xmlns') .remove_default_filter('has_user') .size(0), 'sql_rows': FormData.objects.exclude(domain__isnull=True).count(), }, { 'description': 'Cases (doc_type is "CommCareCase")', 'couch_docs': get_total_case_count(), 'es_query': CaseES().size(0), 'sql_rows': CaseData.objects.exclude(domain__isnull=True).count(), } ] comparisons = [] for comp in comparison_config: comparisons.append({ 'description': comp['description'], 'couch_docs': comp['couch_docs'], 'es_docs': comp['es_query'].run().total, 'sql_rows': comp['sql_rows'] if comp['sql_rows'] else 'n/a', }) return json_response(comparisons)
def update_calculated_properties(): results = DomainES().fields(["name", "_id"]).run().hits all_stats = _all_domain_stats() for r in results: dom = r["name"] try: calced_props = { "_id": r["_id"], "cp_n_web_users": int(all_stats["web_users"].get(dom, 0)), "cp_n_active_cc_users": int(CALC_FNS["mobile_users"](dom)), "cp_n_cc_users": int(all_stats["commcare_users"].get(dom, 0)), "cp_n_active_cases": int(CALC_FNS["cases_in_last"](dom, 120)), "cp_n_users_submitted_form": total_distinct_users([dom]), "cp_n_inactive_cases": int(CALC_FNS["inactive_cases_in_last"](dom, 120)), "cp_n_30_day_cases": int(CALC_FNS["cases_in_last"](dom, 30)), "cp_n_60_day_cases": int(CALC_FNS["cases_in_last"](dom, 60)), "cp_n_90_day_cases": int(CALC_FNS["cases_in_last"](dom, 90)), "cp_n_cases": int(all_stats["cases"].get(dom, 0)), "cp_n_forms": int(all_stats["forms"].get(dom, 0)), "cp_n_forms_30_d": int(CALC_FNS["forms_in_last"](dom, 30)), "cp_n_forms_60_d": int(CALC_FNS["forms_in_last"](dom, 60)), "cp_n_forms_90_d": int(CALC_FNS["forms_in_last"](dom, 90)), "cp_first_form": CALC_FNS["first_form_submission"](dom, False), "cp_last_form": CALC_FNS["last_form_submission"](dom, False), "cp_is_active": CALC_FNS["active"](dom), "cp_has_app": CALC_FNS["has_app"](dom), "cp_last_updated": json_format_datetime(datetime.utcnow()), "cp_n_in_sms": int(CALC_FNS["sms"](dom, "I")), "cp_n_out_sms": int(CALC_FNS["sms"](dom, "O")), "cp_n_sms_ever": int(CALC_FNS["sms_in_last"](dom)), "cp_n_sms_30_d": int(CALC_FNS["sms_in_last"](dom, 30)), "cp_n_sms_60_d": int(CALC_FNS["sms_in_last"](dom, 60)), "cp_n_sms_90_d": int(CALC_FNS["sms_in_last"](dom, 90)), "cp_sms_ever": int(CALC_FNS["sms_in_last_bool"](dom)), "cp_sms_30_d": int(CALC_FNS["sms_in_last_bool"](dom, 30)), "cp_n_sms_in_30_d": int(CALC_FNS["sms_in_in_last"](dom, 30)), "cp_n_sms_in_60_d": int(CALC_FNS["sms_in_in_last"](dom, 60)), "cp_n_sms_in_90_d": int(CALC_FNS["sms_in_in_last"](dom, 90)), "cp_n_sms_out_30_d": int(CALC_FNS["sms_out_in_last"](dom, 30)), "cp_n_sms_out_60_d": int(CALC_FNS["sms_out_in_last"](dom, 60)), "cp_n_sms_out_90_d": int(CALC_FNS["sms_out_in_last"](dom, 90)), "cp_n_j2me_30_d": int(CALC_FNS["j2me_forms_in_last"](dom, 30)), "cp_n_j2me_60_d": int(CALC_FNS["j2me_forms_in_last"](dom, 60)), "cp_n_j2me_90_d": int(CALC_FNS["j2me_forms_in_last"](dom, 90)), "cp_j2me_90_d_bool": int(CALC_FNS["j2me_forms_in_last_bool"](dom, 90)), "cp_300th_form": CALC_FNS["300th_form_submission"](dom) } if calced_props['cp_first_form'] is None: del calced_props['cp_first_form'] if calced_props['cp_last_form'] is None: del calced_props['cp_last_form'] if calced_props['cp_300th_form'] is None: del calced_props['cp_300th_form'] send_to_elasticsearch("domains", calced_props) except Exception, e: notify_exception( None, message='Domain {} failed on stats calculations with {}'. format(dom, e))