def test_unknown_user_pillow(self): FormProcessorTestUtils.delete_all_xforms() user_id = 'test-unknown-user' metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user') form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models( [form]) # send to kafka topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM since = self._get_kafka_seq() producer.send_change(topic, _form_to_change_meta(form)) # send to elasticsearch pillow = get_xform_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # the default query doesn't include unknown users so should have no results self.assertEqual(0, UserES().run().total) # clear the default filters which hide unknown users user_es = UserES().remove_default_filters() results = user_es.run() self.assertEqual(1, results.total) user_doc = results.hits[0] self.assertEqual(TEST_DOMAIN, user_doc['domain']) self.assertEqual(user_id, user_doc['_id']) self.assertEqual('UnknownUser', user_doc['doc_type'])
def setUp(self): super(FormPillowTest, self).setUp() FormProcessorTestUtils.delete_all_xforms() self.pillow = get_xform_pillow(skip_ucr=True) factory = AppFactory(domain=self.domain) self.app = factory.app self.app.save()
def test_unknown_user_pillow(self): FormProcessorTestUtils.delete_all_xforms() user_id = 'test-unknown-user' metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user') form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models([form]) # send to kafka topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM since = self._get_kafka_seq() producer.send_change(topic, _form_to_change_meta(form)) # send to elasticsearch pillow = get_xform_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # the default query doesn't include unknown users so should have no results self.assertEqual(0, UserES().run().total) # clear the default filters which hide unknown users user_es = UserES().remove_default_filters() results = user_es.run() self.assertEqual(1, results.total) user_doc = results.hits[0] self.assertEqual(TEST_DOMAIN, user_doc['domain']) self.assertEqual(user_id, user_doc['_id']) self.assertEqual('UnknownUser', user_doc['doc_type'])
def cleanup_stale_es_on_couch_domains(start_date=None, end_date=None, domains=None, stdout=None): """ This is the response to https://dimagi-dev.atlassian.net/browse/SAAS-11489 and basically makes sure that there are no stale docs in the most active domains still using the couch db backend until we can get them migrated. """ end = end_date or datetime.datetime.utcnow() start = start_date or (end - datetime.timedelta(days=2)) couch_domains = domains or ACTIVE_COUCH_DOMAINS.get_enabled_domains() for domain in couch_domains: form_ids, has_discrepancies = _get_all_form_ids(domain, start, end) if stdout: stdout.write(f"Found {len(form_ids)} in {domain} for between " f"{start.isoformat()} and {end.isoformat()}.") if has_discrepancies: metrics_gauge( 'commcare.es.couch_domain.couch_discrepancy_detected', 1, tags={ 'domain': domain, }) if stdout: stdout.write( f"\tFound discrepancies in form counts for domain {domain}" ) forms_not_in_es = _get_forms_not_in_es(form_ids) if forms_not_in_es: metrics_gauge('commcare.es.couch_domain.stale_docs_in_es', len(forms_not_in_es), tags={ 'domain': domain, }) if stdout: stdout.write(f"\tFound {len(forms_not_in_es)} forms not in es " f"for {domain}") changes = _get_changes(domain, forms_not_in_es) form_es_processor = get_xform_pillow().processors[0] for change in changes: form_es_processor.process_change(change)
def setUpClass(cls): super().setUpClass() cls.domain = 'naboo' cls.case_type = 'people' cls.factory = CaseFactory(cls.domain) cls.pillow = get_xform_pillow(skip_ucr=True) cls.rule = AutomaticUpdateRule.objects.create( domain=cls.domain, name='test', case_type=cls.case_type, active=True, deleted=False, filter_on_server_modified=False, server_modified_boundary=None, workflow=AutomaticUpdateRule.WORKFLOW_DEDUPLICATE, ) _, cls.action = cls.rule.add_action( CaseDeduplicationActionDefinition, match_type=CaseDeduplicationMatchTypeChoices.ALL, case_properties=["case_name", "age"], ) cls.action.set_properties_to_update([ CaseDeduplicationActionDefinition.PropertyDefinition( name='age', value_type=CaseDeduplicationActionDefinition.VALUE_TYPE_EXACT, value='5', ), CaseDeduplicationActionDefinition.PropertyDefinition( name='case_name', value_type=CaseDeduplicationActionDefinition.VALUE_TYPE_EXACT, value='Herman Miller', ) ]) cls.action.save() AutomaticUpdateRule.clear_caches( cls.domain, AutomaticUpdateRule.WORKFLOW_DEDUPLICATE)