def test_unknown_user_pillow(self):
FormProcessorTestUtils.delete_all_xforms()
user_id = 'test-unknown-user'
metadata = TestFormMetadata(domain=TEST_DOMAIN,
user_id='test-unknown-user')
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models(
[form])
# send to kafka
topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM
since = self._get_kafka_seq()
producer.send_change(topic, _form_to_change_meta(form))
# send to elasticsearch
pillow = get_xform_pillow()
pillow.process_changes(since=since, forever=False)
self.elasticsearch.indices.refresh(self.index_info.index)
# the default query doesn't include unknown users so should have no results
self.assertEqual(0, UserES().run().total)
# clear the default filters which hide unknown users
user_es = UserES().remove_default_filters()
results = user_es.run()
self.assertEqual(1, results.total)
user_doc = results.hits[0]
self.assertEqual(TEST_DOMAIN, user_doc['domain'])
self.assertEqual(user_id, user_doc['_id'])
self.assertEqual('UnknownUser', user_doc['doc_type'])
def setUp(self):
super(FormPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_xforms()
self.pillow = get_xform_pillow(skip_ucr=True)
factory = AppFactory(domain=self.domain)
self.app = factory.app
self.app.save()
def test_unknown_user_pillow(self):
FormProcessorTestUtils.delete_all_xforms()
user_id = 'test-unknown-user'
metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user')
form = get_form_ready_to_save(metadata)
FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models([form])
# send to kafka
topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM
since = self._get_kafka_seq()
producer.send_change(topic, _form_to_change_meta(form))
# send to elasticsearch
pillow = get_xform_pillow()
pillow.process_changes(since=since, forever=False)
self.elasticsearch.indices.refresh(self.index_info.index)
# the default query doesn't include unknown users so should have no results
self.assertEqual(0, UserES().run().total)
# clear the default filters which hide unknown users
user_es = UserES().remove_default_filters()
results = user_es.run()
self.assertEqual(1, results.total)
user_doc = results.hits[0]
self.assertEqual(TEST_DOMAIN, user_doc['domain'])
self.assertEqual(user_id, user_doc['_id'])
self.assertEqual('UnknownUser', user_doc['doc_type'])
def setUp(self):
super(FormPillowTest, self).setUp()
FormProcessorTestUtils.delete_all_xforms()
self.pillow = get_xform_pillow(skip_ucr=True)
factory = AppFactory(domain=self.domain)
self.app = factory.app
self.app.save()
def cleanup_stale_es_on_couch_domains(start_date=None,
end_date=None,
domains=None,
stdout=None):
"""
This is the response to https://dimagi-dev.atlassian.net/browse/SAAS-11489
and basically makes sure that there are no stale docs in the most active
domains still using the couch db backend until we can get them migrated.
"""
end = end_date or datetime.datetime.utcnow()
start = start_date or (end - datetime.timedelta(days=2))
couch_domains = domains or ACTIVE_COUCH_DOMAINS.get_enabled_domains()
for domain in couch_domains:
form_ids, has_discrepancies = _get_all_form_ids(domain, start, end)
if stdout:
stdout.write(f"Found {len(form_ids)} in {domain} for between "
f"{start.isoformat()} and {end.isoformat()}.")
if has_discrepancies:
metrics_gauge(
'commcare.es.couch_domain.couch_discrepancy_detected',
1,
tags={
'domain': domain,
})
if stdout:
stdout.write(
f"\tFound discrepancies in form counts for domain {domain}"
)
forms_not_in_es = _get_forms_not_in_es(form_ids)
if forms_not_in_es:
metrics_gauge('commcare.es.couch_domain.stale_docs_in_es',
len(forms_not_in_es),
tags={
'domain': domain,
})
if stdout:
stdout.write(f"\tFound {len(forms_not_in_es)} forms not in es "
f"for {domain}")
changes = _get_changes(domain, forms_not_in_es)
form_es_processor = get_xform_pillow().processors[0]
for change in changes:
form_es_processor.process_change(change)
def setUpClass(cls):
super().setUpClass()
cls.domain = 'naboo'
cls.case_type = 'people'
cls.factory = CaseFactory(cls.domain)
cls.pillow = get_xform_pillow(skip_ucr=True)
cls.rule = AutomaticUpdateRule.objects.create(
domain=cls.domain,
name='test',
case_type=cls.case_type,
active=True,
deleted=False,
filter_on_server_modified=False,
server_modified_boundary=None,
workflow=AutomaticUpdateRule.WORKFLOW_DEDUPLICATE,
)
_, cls.action = cls.rule.add_action(
CaseDeduplicationActionDefinition,
match_type=CaseDeduplicationMatchTypeChoices.ALL,
case_properties=["case_name", "age"],
)
cls.action.set_properties_to_update([
CaseDeduplicationActionDefinition.PropertyDefinition(
name='age',
value_type=CaseDeduplicationActionDefinition.VALUE_TYPE_EXACT,
value='5',
),
CaseDeduplicationActionDefinition.PropertyDefinition(
name='case_name',
value_type=CaseDeduplicationActionDefinition.VALUE_TYPE_EXACT,
value='Herman Miller',
)
])
cls.action.save()
AutomaticUpdateRule.clear_caches(
cls.domain, AutomaticUpdateRule.WORKFLOW_DEDUPLICATE)
