def _update_calculated_properties():
results = DomainES().filter(
get_domains_to_update_es_filter()
).fields(["name", "_id"]).run().hits
all_stats = all_domain_stats()
active_users_by_domain = {}
for r in results:
dom = r["name"]
domain_obj = Domain.get_by_name(dom)
if not domain_obj:
send_to_elasticsearch("domains", r, delete=True)
continue
try:
props = calced_props(domain_obj, r["_id"], all_stats)
active_users_by_domain[dom] = props['cp_n_active_cc_users']
if props['cp_first_form'] is None:
del props['cp_first_form']
if props['cp_last_form'] is None:
del props['cp_last_form']
if props['cp_300th_form'] is None:
del props['cp_300th_form']
send_to_elasticsearch("domains", props, es_merge_update=True)
except Exception as e:
notify_exception(None, message='Domain {} failed on stats calculations with {}'.format(dom, e))
datadog_report_user_stats('commcare.active_mobile_workers.count', active_users_by_domain)
def update_calculated_properties():
domains_to_update = DomainES().filter(
get_domains_to_update_es_filter()
).fields(["name", "_id"]).run().hits
for chunk in chunked(domains_to_update, 5000):
update_calculated_properties_for_domains.delay(chunk)
def test_domain_reindexer(self):
delete_all_domains()
ensure_index_deleted(DOMAIN_INDEX)
name = 'reindex-test-domain'
create_domain(name)
reindex_and_clean('domain')
results = DomainES().run()
self.assertEqual(1, results.total, results.hits)
domain_doc = results.hits[0]
self.assertEqual(name, domain_doc['name'])
self.assertEqual('Domain', domain_doc['doc_type'])
delete_es_index(DOMAIN_INDEX)
def _exclude_missing_domains(configs):
from corehq.apps.es import DomainES
from corehq.elastic import ESError
config_domains = {conf.domain for conf in configs}
try:
domains_present = set(DomainES().in_domains(config_domains).values_list('name', flat=True))
except ESError:
pillow_logging.exception("Unable to filter configs by domain")
return configs
return [config for config in configs if config.domain in domains_present]
def test_domain_reindexer(self):
delete_all_domains()
ensure_index_deleted(DOMAIN_INDEX)
name = 'reindex-test-domain'
create_domain(name)
call_command('ptop_reindexer_v2', 'domain', cleanup=True, noinput=True)
results = DomainES().run()
self.assertEqual(1, results.total, results.hits)
domain_doc = results.hits[0]
self.assertEqual(name, domain_doc['name'])
self.assertEqual('Domain', domain_doc['doc_type'])
delete_es_index(DOMAIN_INDEX)
def couch_sql_migration_stats():
result = (DomainES().filter(filters.term(
'use_sql_backend', False)).remove_default_filters().aggregations([
aggregations.SumAggregation('cases', 'cp_n_cases'),
aggregations.SumAggregation('forms', 'cp_n_forms'),
]).size(0).run())
datadog_gauge('commcare.couch_sql_migration.domains_remaining',
int(result.total))
datadog_gauge('commcare.couch_sql_migration.forms_remaining',
int(result.aggregations.forms.value))
datadog_gauge('commcare.couch_sql_migration.cases_remaining',
int(result.aggregations.cases.value))
def test_kafka_domain_pillow_deletions(self):
# run the other test to ensure domain is created and in ES
self.test_kafka_domain_pillow()
domain_obj = Domain.get_by_name('domain-pillowtest-kafka')
domain_obj.doc_type = 'Domain-DUPLICATE'
# send to kafka
since = get_topic_offset(topics.DOMAIN)
producer.send_change(topics.DOMAIN, _domain_to_change_meta(domain_obj))
# send to elasticsearch
pillow = get_domain_kafka_to_elasticsearch_pillow()
pillow.process_changes(since=since, forever=False)
self.elasticsearch.indices.refresh(self.index_info.index)
# ensure removed from ES
self.assertEqual(0, DomainES().run().total)
def _verify_domain_in_es(self, domain_name):
results = DomainES().run()
self.assertEqual(1, results.total)
domain_doc = results.hits[0]
self.assertEqual(domain_name, domain_doc['name'])
self.assertEqual('Domain', domain_doc['doc_type'])
def get_domains_created_by_user(creating_user):
query = DomainES().created_by_user(creating_user)
data = query.run()
return [d['name'] for d in data.hits]
def get_domains_created_by_user(creating_user):
query = DomainES().created_by_user(creating_user)
data = query.run()
return [d['name'] for d in data.hits]
def get_affected_location_types():
commtrack_domains = (DomainES().commtrack_domains().values_list('name',
flat=True))
return (LocationType.objects.exclude(domain__in=commtrack_domains).filter(
administrative=False))