def test_unknown_user_pillow(self): FormProcessorTestUtils.delete_all_xforms() user_id = 'test-unknown-user' metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user') form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models([form]) # send to kafka topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM since = self._get_kafka_seq() producer.send_change(topic, _form_to_change_meta(form)) # send to elasticsearch pillow = get_unknown_users_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # the default query doesn't include unknown users so should have no results self.assertEqual(0, UserES().run().total) # clear the default filters which hide unknown users user_es = UserES().remove_default_filters() results = user_es.run() self.assertEqual(1, results.total) user_doc = results.hits[0] self.assertEqual(TEST_DOMAIN, user_doc['domain']) self.assertEqual(user_id, user_doc['_id']) self.assertEqual('UnknownUser', user_doc['doc_type'])
def test_xform_pillow_sql(self): consumer = get_test_kafka_consumer(topics.FORM_SQL) # have to get the seq id before the change is processed kafka_seq = consumer.offsets()['fetch'][(topics.FORM_SQL, 0)] metadata = TestFormMetadata(domain=self.domain) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=self.domain) form_processor.save_processed_models([form]) # confirm change made it to kafka message = consumer.next() change_meta = change_meta_from_kafka_message(message.value) self.assertEqual(form.form_id, change_meta.document_id) self.assertEqual(self.domain, change_meta.domain) # send to elasticsearch sql_pillow = get_sql_xform_to_elasticsearch_pillow() sql_pillow.process_changes(since=kafka_seq, forever=False) self.elasticsearch.indices.refresh(self.pillow.es_index) # confirm change made it to elasticserach results = FormES().run() self.assertEqual(1, results.total) form_doc = results.hits[0] self.assertEqual(self.domain, form_doc['domain']) self.assertEqual(metadata.xmlns, form_doc['xmlns']) self.assertEqual('XFormInstance', form_doc['doc_type'])
def _create_form_and_sync_to_es(self): with self.process_form_changes: form = get_form_ready_to_save(self.metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=self.domain) form_processor.save_processed_models([form]) self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index) return form, self.metadata
def test_unknown_user_pillow(self): FormProcessorTestUtils.delete_all_xforms() user_id = 'test-unknown-user' metadata = TestFormMetadata(domain=TEST_DOMAIN, user_id='test-unknown-user') form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=TEST_DOMAIN).save_processed_models( [form]) # send to kafka topic = topics.FORM_SQL if settings.TESTS_SHOULD_USE_SQL_BACKEND else topics.FORM since = self._get_kafka_seq() producer.send_change(topic, _form_to_change_meta(form)) # send to elasticsearch pillow = get_xform_pillow() pillow.process_changes(since=since, forever=False) self.elasticsearch.indices.refresh(self.index_info.index) # the default query doesn't include unknown users so should have no results self.assertEqual(0, UserES().run().total) # clear the default filters which hide unknown users user_es = UserES().remove_default_filters() results = user_es.run() self.assertEqual(1, results.total) user_doc = results.hits[0] self.assertEqual(TEST_DOMAIN, user_doc['domain']) self.assertEqual(user_id, user_doc['_id']) self.assertEqual('UnknownUser', user_doc['doc_type'])
def _create_form_and_sync_to_es(self, domain): with process_kafka_changes('ReportXFormToElasticsearchPillow'): with process_couch_changes('DefaultChangeFeedPillow'): metadata = TestFormMetadata(domain=domain) form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=domain).save_processed_models([form]) self.elasticsearch.indices.refresh(REPORT_XFORM_INDEX_INFO.index) return form, metadata
def _create_form_and_sync_to_es(self, domain): with process_pillow_changes('ReportXFormToElasticsearchPillow'): with process_pillow_changes('DefaultChangeFeedPillow'): metadata = TestFormMetadata(domain=domain) form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=domain).save_processed_models([form]) self.elasticsearch.indices.refresh(REPORT_XFORM_INDEX_INFO.index) return form, metadata
def create_form_and_sync_to_es(received_on): with process_pillow_changes('xform-pillow', {'skip_ucr': True}): with process_pillow_changes('DefaultChangeFeedPillow'): metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id, xmlns=cls.xmlns, received_on=received_on) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=cls.domain) form_processor.save_processed_models([form]) return form
def _create_form_and_sync_to_es(self): with process_pillow_changes('xform-pillow', {'skip_ucr': True}): with process_pillow_changes('DefaultChangeFeedPillow'): metadata = TestFormMetadata(domain=self.domain) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=self.domain) form_processor.save_processed_models([form]) self.elasticsearch.indices.refresh(XFORM_INDEX_INFO.index) return form, metadata
def create_form_and_sync_to_es(received_on): with process_kafka_changes('XFormToElasticsearchPillow'): with process_couch_changes('DefaultChangeFeedPillow'): metadata = TestFormMetadata(domain=cls.domain, app_id=cls.app_id, xmlns=cls.xmlns, received_on=received_on) form = get_form_ready_to_save(metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=cls.domain) form_processor.save_processed_models([form]) return form
def test_xform_reindexers(self): FormProcessorTestUtils.delete_all_xforms() metadata = TestFormMetadata(domain=self.domain) form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=self.domain).save_processed_models([form]) call_command('ptop_fast_reindex_xforms', noinput=True, bulk=True) results = FormES().run() self.assertEqual(1, results.total) form_doc = results.hits[0] self.assertEqual(self.domain, form_doc['domain']) self.assertEqual(metadata.xmlns, form_doc['xmlns']) self.assertEqual('XFormInstance', form_doc['doc_type']) form.delete()
def test_xform_pillow_couch(self): metadata = TestFormMetadata(domain=self.domain) form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=self.domain).save_processed_models([form]) self.pillow.process_changes(since=0, forever=False) self.elasticsearch.indices.refresh(self.pillow.es_index) results = FormES().run() self.assertEqual(1, results.total) form_doc = results.hits[0] self.assertEqual(self.domain, form_doc['domain']) self.assertEqual(metadata.xmlns, form_doc['xmlns']) self.assertEqual('XFormInstance', form_doc['doc_type']) form.delete()
def _create_form(self, user_id=None, received_on=None): """ Create the models directly so that these tests aren't dependent on any other apps. :return: XFormInstanceSQL """ user_id = user_id or 'mr_wednesday' received_on = received_on or datetime.utcnow() metadata = TestFormMetadata(domain=self.DOMAIN, received_on=received_on, user_id=user_id) form = get_form_ready_to_save(metadata) return form
def _create_form(self, user_id=None, received_on=None): """ Create the models directly so that these tests aren't dependent on any other apps. :return: XFormInstanceSQL """ user_id = user_id or 'mr_wednesday' received_on = received_on or datetime.utcnow() metadata = TestFormMetadata( domain=self.DOMAIN, received_on=received_on, user_id=user_id ) form = get_form_ready_to_save(metadata) return form
def test_unknown_user_reindexer(self): FormProcessorTestUtils.delete_all_xforms() user_id = 'test-unknown-user' metadata = TestFormMetadata(domain=self.domain, user_id='test-unknown-user') form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=self.domain).save_processed_models([form]) ensure_index_deleted(USER_INDEX) call_command('ptop_fast_reindex_unknownusers', noinput=True, bulk=True) # the default query doesn't include unknown users so should have no results self.assertEqual(0, UserES().run().total) user_es = UserES() # hack: clear the default filters which hide unknown users # todo: find a better way to do this. user_es._default_filters = ESQuery.default_filters results = user_es.run() self.assertEqual(1, results.total) user_doc = results.hits[0] self.assertEqual(self.domain, user_doc['domain']) self.assertEqual(user_id, user_doc['_id']) self.assertEqual('UnknownUser', user_doc['doc_type']) form.delete() delete_es_index(USER_INDEX)
def get_form(form_id): metadata = TestFormMetadata(domain="test", user_id=USER_ID) return get_form_ready_to_save(metadata, form_id=form_id)
def _create_form(self, domain): metadata = TestFormMetadata(domain=domain) form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=domain).save_processed_models([form]) return form
def _create_and_save_a_form(): metadata = TestFormMetadata(domain=DOMAIN) form = get_form_ready_to_save(metadata) FormProcessorInterface(domain=DOMAIN).save_processed_models([form]) return form
def _create_form(self): form = get_form_ready_to_save(self.metadata, is_db_test=True) form_processor = FormProcessorInterface(domain=self.domain) form_processor.save_processed_models([form]) return form