Example #1
0
def _update_calculated_properties():
    results = DomainES().filter(
        get_domains_to_update_es_filter()
    ).fields(["name", "_id"]).run().hits

    all_stats = all_domain_stats()

    active_users_by_domain = {}
    for r in results:
        dom = r["name"]
        domain_obj = Domain.get_by_name(dom)
        if not domain_obj:
            send_to_elasticsearch("domains", r, delete=True)
            continue
        try:
            props = calced_props(domain_obj, r["_id"], all_stats)
            active_users_by_domain[dom] = props['cp_n_active_cc_users']
            if props['cp_first_form'] is None:
                del props['cp_first_form']
            if props['cp_last_form'] is None:
                del props['cp_last_form']
            if props['cp_300th_form'] is None:
                del props['cp_300th_form']
            send_to_elasticsearch("domains", props, es_merge_update=True)
        except Exception as e:
            notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))

    datadog_report_user_stats('commcare.active_mobile_workers.count', active_users_by_domain)
Example #2
0
def update_calculated_properties():
    domains_to_update = DomainES().filter(
        get_domains_to_update_es_filter()
    ).fields(["name", "_id"]).run().hits

    for chunk in chunked(domains_to_update, 5000):
        update_calculated_properties_for_domains.delay(chunk)
Example #3
0
 def test_domain_reindexer(self):
     delete_all_domains()
     ensure_index_deleted(DOMAIN_INDEX)
     name = 'reindex-test-domain'
     create_domain(name)
     reindex_and_clean('domain')
     results = DomainES().run()
     self.assertEqual(1, results.total, results.hits)
     domain_doc = results.hits[0]
     self.assertEqual(name, domain_doc['name'])
     self.assertEqual('Domain', domain_doc['doc_type'])
     delete_es_index(DOMAIN_INDEX)
Example #4
0
def _exclude_missing_domains(configs):
    from corehq.apps.es import DomainES
    from corehq.elastic import ESError

    config_domains = {conf.domain for conf in configs}
    try:
        domains_present = set(DomainES().in_domains(config_domains).values_list('name', flat=True))
    except ESError:
        pillow_logging.exception("Unable to filter configs by domain")
        return configs

    return [config for config in configs if config.domain in domains_present]
 def test_domain_reindexer(self):
     delete_all_domains()
     ensure_index_deleted(DOMAIN_INDEX)
     name = 'reindex-test-domain'
     create_domain(name)
     call_command('ptop_reindexer_v2', 'domain', cleanup=True, noinput=True)
     results = DomainES().run()
     self.assertEqual(1, results.total, results.hits)
     domain_doc = results.hits[0]
     self.assertEqual(name, domain_doc['name'])
     self.assertEqual('Domain', domain_doc['doc_type'])
     delete_es_index(DOMAIN_INDEX)
Example #6
0
def couch_sql_migration_stats():
    result = (DomainES().filter(filters.term(
        'use_sql_backend', False)).remove_default_filters().aggregations([
            aggregations.SumAggregation('cases', 'cp_n_cases'),
            aggregations.SumAggregation('forms', 'cp_n_forms'),
        ]).size(0).run())

    datadog_gauge('commcare.couch_sql_migration.domains_remaining',
                  int(result.total))
    datadog_gauge('commcare.couch_sql_migration.forms_remaining',
                  int(result.aggregations.forms.value))
    datadog_gauge('commcare.couch_sql_migration.cases_remaining',
                  int(result.aggregations.cases.value))
Example #7
0
    def test_kafka_domain_pillow_deletions(self):
        # run the other test to ensure domain is created and in ES
        self.test_kafka_domain_pillow()
        domain_obj = Domain.get_by_name('domain-pillowtest-kafka')
        domain_obj.doc_type = 'Domain-DUPLICATE'

        # send to kafka
        since = get_topic_offset(topics.DOMAIN)
        producer.send_change(topics.DOMAIN, _domain_to_change_meta(domain_obj))

        # send to elasticsearch
        pillow = get_domain_kafka_to_elasticsearch_pillow()
        pillow.process_changes(since=since, forever=False)
        self.elasticsearch.indices.refresh(self.index_info.index)

        # ensure removed from ES
        self.assertEqual(0, DomainES().run().total)
Example #8
0
 def _verify_domain_in_es(self, domain_name):
     results = DomainES().run()
     self.assertEqual(1, results.total)
     domain_doc = results.hits[0]
     self.assertEqual(domain_name, domain_doc['name'])
     self.assertEqual('Domain', domain_doc['doc_type'])
Example #9
0
def get_domains_created_by_user(creating_user):
    query = DomainES().created_by_user(creating_user)
    data = query.run()
    return [d['name'] for d in data.hits]
Example #10
0
def get_domains_created_by_user(creating_user):
    query = DomainES().created_by_user(creating_user)
    data = query.run()
    return [d['name'] for d in data.hits]
def get_affected_location_types():
    commtrack_domains = (DomainES().commtrack_domains().values_list('name',
                                                                    flat=True))
    return (LocationType.objects.exclude(domain__in=commtrack_domains).filter(
        administrative=False))